feat: merge Xid-Catalog.xlsx, fix dmesg regex, and improve history UI
This commit is contained in:
unknown
@@ -0,0 +1,6 @@
|
||||
__pycache__
|
||||
*.pyc
|
||||
.git
|
||||
.gitignore
|
||||
*.md
|
||||
.env
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
.Python
|
||||
env/
|
||||
venv/
|
||||
.venv/
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Databases
|
||||
data/nv_log_history.db
|
||||
|
||||
# Backup files
|
||||
*.bak
|
||||
|
||||
# IDEs and editors
|
||||
.vscode/
|
||||
.idea/
|
||||
.claude/
|
||||
*.suo
|
||||
*.ntvs*
|
||||
*.njsproj
|
||||
*.sln
|
||||
*.swp
|
||||
|
||||
# OS generated files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
@@ -0,0 +1,2 @@
|
||||
[client]
|
||||
toolbarMode = "minimal"
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 시스템 의존성 설치
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 파이썬 패키지 설치
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# 애플리케이션 복사
|
||||
COPY . .
|
||||
|
||||
# Streamlit 포트
|
||||
EXPOSE 8501
|
||||
|
||||
# 헬스체크
|
||||
HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health || exit 1
|
||||
|
||||
# Streamlit 실행
|
||||
ENTRYPOINT ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0", "--server.headless=true"]
|
||||
@@ -0,0 +1,79 @@
|
||||
# NV-Log Analyzer
|
||||
|
||||
NVIDIA Bug Report 로그 자가 진단 서비스
|
||||
|
||||
`nvidia-bug-report.log.gz` 파일을 업로드하면 XID 에러, GPU 상태, 커널 로그를 자동 분석하여 권장 조치를 안내합니다.
|
||||
|
||||
## 주요 기능
|
||||
|
||||
- **XID 에러 탐지**: `NVRM: Xid` 패턴 자동 검출 + 에러 코드별 한글 권장 조치 제공
|
||||
- **GPU 헬스 체크**: 온도, 전력, ECC 에러, 메모리 상태 요약
|
||||
- **커널 로그 필터링**: dmesg 내 Critical, Error, OOM, PCIe 에러 추출
|
||||
- **원본 로그 검색**: 키워드 기반 로그 검색 기능
|
||||
- **XID 코드 조회**: 사이드바에서 XID 번호로 즉시 의미 확인
|
||||
|
||||
## 빠른 시작
|
||||
|
||||
### 방법 1: 로컬 실행
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
streamlit run app.py
|
||||
```
|
||||
|
||||
브라우저에서 `http://localhost:8501` 접속
|
||||
|
||||
### 방법 2: Docker
|
||||
|
||||
```bash
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
브라우저에서 `http://localhost:8501` 접속
|
||||
|
||||
## 프로젝트 구조
|
||||
|
||||
```
|
||||
nvidia/
|
||||
├── app.py # Streamlit 대시보드 (메인 UI)
|
||||
├── log_parser.py # 핵심 분석 엔진
|
||||
├── data/
|
||||
│ └── xid_database.json # XID 에러 코드 매핑 DB (18종)
|
||||
├── requirements.txt
|
||||
├── Dockerfile
|
||||
├── docker-compose.yml
|
||||
└── README.md
|
||||
```
|
||||
|
||||
## 분석 대상 로그 생성 방법
|
||||
|
||||
GPU 서버에서 아래 명령 실행:
|
||||
|
||||
```bash
|
||||
sudo nvidia-bug-report.sh
|
||||
```
|
||||
|
||||
생성된 `nvidia-bug-report.log.gz` 파일을 웹 대시보드에 업로드하면 됩니다.
|
||||
|
||||
## 지원하는 XID 코드
|
||||
|
||||
| XID | 이름 | 심각도 |
|
||||
|-----|------|--------|
|
||||
| 8 | GPU Stopped Responding | 심각 |
|
||||
| 13 | Graphics Engine Exception | 심각 |
|
||||
| 31 | GPU Memory Page Fault | 심각 |
|
||||
| 32 | Invalid Push Buffer | 경고 |
|
||||
| 38 | Driver Firmware Mismatch | 경고 |
|
||||
| 43 | GPU Stopped Processing | 심각 |
|
||||
| 45 | GPU Preemption Failure | 경고 |
|
||||
| 48 | Double Bit ECC Error | 심각 |
|
||||
| 61 | GPU Fallen Off the Bus | 심각 |
|
||||
| 62 | GPU ECC Page Retirement | 경고 |
|
||||
| 63 | ECC Page Retirement: Double Bit | 심각 |
|
||||
| 64 | ECC Page Retirement Failure | 심각 |
|
||||
| 69 | GPU Access to System Memory Failed | 경고 |
|
||||
| 79 | GPU Fallen Off the Bus (Fatal) | 심각 |
|
||||
| 92 | High Single-bit ECC Error Rate | 경고 |
|
||||
| 94 | Contained ECC Error | 정보 |
|
||||
| 95 | Uncontained ECC Error | 심각 |
|
||||
| 119 | GPU Recovery Action | 정보 |
|
||||
Binary file not shown.
@@ -0,0 +1,204 @@
|
||||
"""
|
||||
NV-Log Analyzer - 인증 및 사용자 관리 모듈
|
||||
역할: admin(관리자), editor(편집자), viewer(뷰어)
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import secrets
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
DB_PATH = Path(__file__).parent / "data" / "nv_log_history.db"
|
||||
|
||||
ROLES = {
|
||||
"admin": {
|
||||
"label": "관리자",
|
||||
"description": "모든 기능 사용 가능 + 사용자 관리",
|
||||
"can_upload": True,
|
||||
"can_view": True,
|
||||
"can_edit_memo": True,
|
||||
"can_delete": True,
|
||||
"can_manage_users": True,
|
||||
},
|
||||
"editor": {
|
||||
"label": "편집자",
|
||||
"description": "로그 업로드, 분석 결과 조회/수정 가능",
|
||||
"can_upload": True,
|
||||
"can_view": True,
|
||||
"can_edit_memo": True,
|
||||
"can_delete": False,
|
||||
"can_manage_users": False,
|
||||
},
|
||||
"viewer": {
|
||||
"label": "뷰어",
|
||||
"description": "분석 결과 조회만 가능",
|
||||
"can_upload": False,
|
||||
"can_view": True,
|
||||
"can_edit_memo": False,
|
||||
"can_delete": False,
|
||||
"can_manage_users": False,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_conn():
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
try:
|
||||
yield conn
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _hash_password(password: str, salt: str = None) -> tuple:
|
||||
"""비밀번호 해싱 (SHA-256 + salt)"""
|
||||
if salt is None:
|
||||
salt = secrets.token_hex(16)
|
||||
hashed = hashlib.sha256((salt + password).encode()).hexdigest()
|
||||
return hashed, salt
|
||||
|
||||
|
||||
def init_user_db():
|
||||
"""사용자 테이블 초기화 + 기본 admin 계정 생성"""
|
||||
with get_conn() as conn:
|
||||
conn.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
password TEXT NOT NULL,
|
||||
salt TEXT NOT NULL,
|
||||
display_name TEXT DEFAULT '',
|
||||
role TEXT NOT NULL DEFAULT 'viewer',
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now','localtime')),
|
||||
approved_by TEXT DEFAULT NULL,
|
||||
last_login TEXT DEFAULT NULL
|
||||
);
|
||||
""")
|
||||
|
||||
# 기본 admin 계정 (최초 1회만)
|
||||
existing = conn.execute("SELECT id FROM users WHERE username = 'admin'").fetchone()
|
||||
if not existing:
|
||||
hashed, salt = _hash_password("admin1234")
|
||||
conn.execute("""
|
||||
INSERT INTO users (username, password, salt, display_name, role, status)
|
||||
VALUES (?, ?, ?, ?, 'admin', 'approved')
|
||||
""", ("admin", hashed, salt, "시스템 관리자"))
|
||||
|
||||
|
||||
def authenticate(username: str, password: str) -> dict:
|
||||
"""로그인 인증. 성공 시 사용자 정보 dict 반환, 실패 시 None (대소문자 무시 체크)"""
|
||||
with get_conn() as conn:
|
||||
user = conn.execute(
|
||||
"SELECT * FROM users WHERE LOWER(username) = LOWER(?)", (username,)
|
||||
).fetchone()
|
||||
|
||||
if not user:
|
||||
return None
|
||||
|
||||
hashed, _ = _hash_password(password, user["salt"])
|
||||
if hashed != user["password"]:
|
||||
return None
|
||||
|
||||
if user["status"] != "approved":
|
||||
return {"error": "pending", "status": user["status"]}
|
||||
|
||||
# 마지막 로그인 시간 업데이트
|
||||
conn.execute(
|
||||
"UPDATE users SET last_login = datetime('now','localtime') WHERE id = ?",
|
||||
(user["id"],)
|
||||
)
|
||||
|
||||
return dict(user)
|
||||
|
||||
|
||||
def register_user(username: str, password: str, display_name: str) -> dict:
|
||||
"""회원가입. 성공 시 {"success": True}, 실패 시 에러 메시지 (대소문자 무시 중복 검사)"""
|
||||
if len(username) < 3:
|
||||
return {"success": False, "error": "아이디는 3자 이상이어야 합니다."}
|
||||
if len(password) < 6:
|
||||
return {"success": False, "error": "비밀번호는 6자 이상이어야 합니다."}
|
||||
|
||||
with get_conn() as conn:
|
||||
existing = conn.execute(
|
||||
"SELECT id FROM users WHERE LOWER(username) = LOWER(?)", (username,)
|
||||
).fetchone()
|
||||
if existing:
|
||||
return {"success": False, "error": "이미 사용 중인 아이디입니다."}
|
||||
|
||||
hashed, salt = _hash_password(password)
|
||||
conn.execute("""
|
||||
INSERT INTO users (username, password, salt, display_name, role, status)
|
||||
VALUES (?, ?, ?, ?, 'viewer', 'pending')
|
||||
""", (username, hashed, salt, display_name))
|
||||
|
||||
return {"success": True}
|
||||
|
||||
|
||||
def get_pending_users() -> list:
|
||||
"""승인 대기 중인 사용자 목록"""
|
||||
with get_conn() as conn:
|
||||
return [dict(r) for r in conn.execute(
|
||||
"SELECT * FROM users WHERE status = 'pending' ORDER BY created_at DESC"
|
||||
).fetchall()]
|
||||
|
||||
|
||||
def get_all_users() -> list:
|
||||
"""전체 사용자 목록"""
|
||||
with get_conn() as conn:
|
||||
return [dict(r) for r in conn.execute(
|
||||
"SELECT * FROM users ORDER BY CASE status WHEN 'pending' THEN 0 ELSE 1 END, created_at DESC"
|
||||
).fetchall()]
|
||||
|
||||
|
||||
def approve_user(user_id: int, role: str, approved_by: str):
|
||||
"""사용자 승인 + 역할 지정"""
|
||||
with get_conn() as conn:
|
||||
conn.execute(
|
||||
"UPDATE users SET status = 'approved', role = ?, approved_by = ? WHERE id = ?",
|
||||
(role, approved_by, user_id)
|
||||
)
|
||||
|
||||
|
||||
def reject_user(user_id: int):
|
||||
"""사용자 가입 거절"""
|
||||
with get_conn() as conn:
|
||||
conn.execute("UPDATE users SET status = 'rejected' WHERE id = ?", (user_id,))
|
||||
|
||||
|
||||
def update_user_role(user_id: int, new_role: str):
|
||||
"""사용자 역할 변경"""
|
||||
with get_conn() as conn:
|
||||
conn.execute("UPDATE users SET role = ? WHERE id = ?", (new_role, user_id))
|
||||
|
||||
|
||||
def delete_user(user_id: int):
|
||||
"""사용자 삭제"""
|
||||
with get_conn() as conn:
|
||||
conn.execute("DELETE FROM users WHERE id = ?", (user_id,))
|
||||
|
||||
|
||||
def update_password(user_id: int, new_password: str) -> bool:
|
||||
"""비밀번호 변경"""
|
||||
if len(new_password) < 6:
|
||||
return False
|
||||
hashed, salt = _hash_password(new_password)
|
||||
with get_conn() as conn:
|
||||
conn.execute(
|
||||
"UPDATE users SET password = ?, salt = ? WHERE id = ?",
|
||||
(hashed, salt, user_id)
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def get_user_permissions(role: str) -> dict:
|
||||
"""역할별 권한 반환"""
|
||||
return ROLES.get(role, ROLES["viewer"])
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,412 @@
|
||||
"""
|
||||
NV-Log Analyzer - SQLite 데이터베이스 모듈
|
||||
분석 결과를 영구 저장하여 장애 케이스 이력을 축적합니다.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
DB_PATH = Path(__file__).parent / "data" / "nv_log_history.db"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_conn():
|
||||
"""DB 커넥션 컨텍스트 매니저"""
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
try:
|
||||
yield conn
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def init_db():
|
||||
"""DB 테이블 초기화 (앱 시작 시 1회 호출)"""
|
||||
with get_conn() as conn:
|
||||
conn.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS analysis (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now','localtime')),
|
||||
filename TEXT NOT NULL,
|
||||
file_size_mb REAL,
|
||||
overall_status TEXT,
|
||||
total_issues INTEGER DEFAULT 0,
|
||||
critical_count INTEGER DEFAULT 0,
|
||||
warning_count INTEGER DEFAULT 0,
|
||||
info_count INTEGER DEFAULT 0,
|
||||
driver_version TEXT,
|
||||
cuda_version TEXT,
|
||||
gpu_count INTEGER DEFAULT 0,
|
||||
xid_count INTEGER DEFAULT 0,
|
||||
kernel_event_count INTEGER DEFAULT 0,
|
||||
rma_count INTEGER DEFAULT 0,
|
||||
ecc_info TEXT,
|
||||
memo TEXT DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS analysis_gpu (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
analysis_id INTEGER NOT NULL REFERENCES analysis(id) ON DELETE CASCADE,
|
||||
gpu_index INTEGER,
|
||||
name TEXT,
|
||||
serial_number TEXT,
|
||||
gpu_uuid TEXT,
|
||||
pci_address TEXT,
|
||||
product_brand TEXT,
|
||||
product_architecture TEXT,
|
||||
driver_version TEXT,
|
||||
cuda_version TEXT,
|
||||
vbios_version TEXT,
|
||||
inforom_version TEXT,
|
||||
temperature INTEGER,
|
||||
temperature_limit TEXT,
|
||||
power_draw REAL,
|
||||
power_limit REAL,
|
||||
power_state TEXT,
|
||||
memory_used INTEGER,
|
||||
memory_total INTEGER,
|
||||
memory_free INTEGER,
|
||||
gpu_utilization INTEGER,
|
||||
memory_utilization INTEGER,
|
||||
fan_speed INTEGER,
|
||||
persistence_mode TEXT,
|
||||
compute_mode TEXT,
|
||||
mig_mode TEXT,
|
||||
ecc_mode TEXT,
|
||||
pci_link_gen TEXT,
|
||||
pci_link_width TEXT,
|
||||
clock_graphics TEXT,
|
||||
clock_memory TEXT,
|
||||
-- RMA 관련
|
||||
remap_corr_error INTEGER DEFAULT 0,
|
||||
remap_uncorr_error INTEGER DEFAULT 0,
|
||||
remap_pending TEXT,
|
||||
remap_failure TEXT,
|
||||
remap_bank_max INTEGER DEFAULT 0,
|
||||
remap_bank_high INTEGER DEFAULT 0,
|
||||
remap_bank_partial INTEGER DEFAULT 0,
|
||||
remap_bank_low INTEGER DEFAULT 0,
|
||||
remap_bank_none INTEGER DEFAULT 0,
|
||||
sram_corr_volatile TEXT,
|
||||
sram_uncorr_parity_volatile TEXT,
|
||||
sram_uncorr_secded_volatile TEXT,
|
||||
sram_corr_aggregate TEXT,
|
||||
sram_uncorr_parity_aggregate TEXT,
|
||||
sram_uncorr_secded_aggregate TEXT,
|
||||
sram_threshold_exceeded TEXT,
|
||||
sram_l2 TEXT,
|
||||
sram_sm TEXT,
|
||||
sram_microcontroller TEXT,
|
||||
sram_pcie TEXT,
|
||||
sram_other TEXT,
|
||||
dram_corr_volatile TEXT,
|
||||
dram_uncorr_volatile TEXT,
|
||||
dram_corr_aggregate TEXT,
|
||||
dram_uncorr_aggregate TEXT,
|
||||
-- HW Slowdown
|
||||
hw_slowdown TEXT,
|
||||
hw_thermal_slowdown TEXT,
|
||||
hw_power_brake_slowdown TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS analysis_rma (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
analysis_id INTEGER NOT NULL REFERENCES analysis(id) ON DELETE CASCADE,
|
||||
gpu_index INTEGER,
|
||||
gpu_name TEXT,
|
||||
serial_number TEXT,
|
||||
qualifies_for_rma INTEGER DEFAULT 0,
|
||||
reasons TEXT,
|
||||
warnings TEXT,
|
||||
row_remap_details TEXT,
|
||||
sram_details TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS analysis_xid (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
analysis_id INTEGER NOT NULL REFERENCES analysis(id) ON DELETE CASCADE,
|
||||
timestamp TEXT,
|
||||
xid_code INTEGER,
|
||||
xid_name TEXT,
|
||||
severity TEXT,
|
||||
pci_address TEXT,
|
||||
gpu_name TEXT,
|
||||
gpu_serial TEXT,
|
||||
description TEXT,
|
||||
action TEXT,
|
||||
raw_line TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS analysis_kernel (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
analysis_id INTEGER NOT NULL REFERENCES analysis(id) ON DELETE CASCADE,
|
||||
timestamp TEXT,
|
||||
level TEXT,
|
||||
message TEXT,
|
||||
raw_line TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_analysis_created ON analysis(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_analysis_status ON analysis(overall_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_xid_code ON analysis_xid(xid_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_xid_severity ON analysis_xid(severity);
|
||||
CREATE INDEX IF NOT EXISTS idx_gpu_serial ON analysis_gpu(serial_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_kernel_level ON analysis_kernel(level);
|
||||
CREATE INDEX IF NOT EXISTS idx_rma_serial ON analysis_rma(serial_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_rma_qualifies ON analysis_rma(qualifies_for_rma);
|
||||
""")
|
||||
# 기존 DB 마이그레이션: gpu_name 컬럼이 없으면 추가
|
||||
cols = {row[1] for row in conn.execute("PRAGMA table_info(analysis_xid)")}
|
||||
if "gpu_name" not in cols:
|
||||
conn.execute("ALTER TABLE analysis_xid ADD COLUMN gpu_name TEXT")
|
||||
if "gpu_serial" not in cols:
|
||||
conn.execute("ALTER TABLE analysis_xid ADD COLUMN gpu_serial TEXT")
|
||||
# HW slowdown 마이그레이션
|
||||
gpu_cols = {row[1] for row in conn.execute("PRAGMA table_info(analysis_gpu)")}
|
||||
if "hw_slowdown" not in gpu_cols:
|
||||
conn.execute("ALTER TABLE analysis_gpu ADD COLUMN hw_slowdown TEXT")
|
||||
if "hw_thermal_slowdown" not in gpu_cols:
|
||||
conn.execute("ALTER TABLE analysis_gpu ADD COLUMN hw_thermal_slowdown TEXT")
|
||||
if "hw_power_brake_slowdown" not in gpu_cols:
|
||||
conn.execute("ALTER TABLE analysis_gpu ADD COLUMN hw_power_brake_slowdown TEXT")
|
||||
|
||||
|
||||
def save_analysis(result) -> int:
|
||||
"""분석 결과 전체를 DB에 저장하고 analysis_id 반환"""
|
||||
with get_conn() as conn:
|
||||
cur = conn.execute("""
|
||||
INSERT INTO analysis
|
||||
(filename, file_size_mb, overall_status,
|
||||
total_issues, critical_count, warning_count, info_count,
|
||||
driver_version, cuda_version, gpu_count,
|
||||
xid_count, kernel_event_count, rma_count, ecc_info)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
""", (
|
||||
result.filename,
|
||||
result.file_size_mb,
|
||||
result.overall_status,
|
||||
result.total_issues,
|
||||
result.critical_count,
|
||||
result.warning_count,
|
||||
result.info_count,
|
||||
result.driver_version,
|
||||
result.cuda_version,
|
||||
len(result.gpus),
|
||||
len(result.xid_events),
|
||||
len(result.kernel_events),
|
||||
result.rma_count,
|
||||
json.dumps(result.ecc_info, ensure_ascii=False) if result.ecc_info else "{}",
|
||||
))
|
||||
analysis_id = cur.lastrowid
|
||||
|
||||
# GPU 정보 저장
|
||||
for gpu in result.gpus:
|
||||
conn.execute("""
|
||||
INSERT INTO analysis_gpu
|
||||
(analysis_id, gpu_index, name, serial_number, gpu_uuid,
|
||||
pci_address, product_brand, product_architecture,
|
||||
driver_version, cuda_version, vbios_version, inforom_version,
|
||||
temperature, temperature_limit, power_draw, power_limit, power_state,
|
||||
memory_used, memory_total, memory_free,
|
||||
gpu_utilization, memory_utilization,
|
||||
fan_speed, persistence_mode, compute_mode, mig_mode, ecc_mode,
|
||||
pci_link_gen, pci_link_width, clock_graphics, clock_memory,
|
||||
remap_corr_error, remap_uncorr_error, remap_pending, remap_failure,
|
||||
remap_bank_max, remap_bank_high, remap_bank_partial, remap_bank_low, remap_bank_none,
|
||||
sram_corr_volatile, sram_uncorr_parity_volatile, sram_uncorr_secded_volatile,
|
||||
sram_corr_aggregate, sram_uncorr_parity_aggregate, sram_uncorr_secded_aggregate,
|
||||
sram_threshold_exceeded,
|
||||
sram_l2, sram_sm, sram_microcontroller, sram_pcie, sram_other,
|
||||
dram_corr_volatile, dram_uncorr_volatile, dram_corr_aggregate, dram_uncorr_aggregate,
|
||||
hw_slowdown, hw_thermal_slowdown, hw_power_brake_slowdown)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,
|
||||
?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
""", (
|
||||
analysis_id, gpu.index, gpu.name, gpu.serial_number, gpu.gpu_uuid,
|
||||
gpu.pci_address, gpu.product_brand, gpu.product_architecture,
|
||||
gpu.driver_version, gpu.cuda_version, gpu.vbios_version, gpu.inforom_version,
|
||||
gpu.temperature, gpu.temperature_limit, gpu.power_draw, gpu.power_limit, gpu.power_state,
|
||||
gpu.memory_used, gpu.memory_total, gpu.memory_free,
|
||||
gpu.gpu_utilization, gpu.memory_utilization,
|
||||
gpu.fan_speed, gpu.persistence_mode, gpu.compute_mode, gpu.mig_mode, gpu.ecc_mode,
|
||||
gpu.pci_link_gen_current, gpu.pci_link_width_current,
|
||||
gpu.clock_graphics, gpu.clock_memory,
|
||||
gpu.remap_corr_error, gpu.remap_uncorr_error, gpu.remap_pending, gpu.remap_failure,
|
||||
gpu.remap_bank_max, gpu.remap_bank_high, gpu.remap_bank_partial, gpu.remap_bank_low, gpu.remap_bank_none,
|
||||
gpu.sram_corr_volatile, gpu.sram_uncorr_parity_volatile, gpu.sram_uncorr_secded_volatile,
|
||||
gpu.sram_corr_aggregate, gpu.sram_uncorr_parity_aggregate, gpu.sram_uncorr_secded_aggregate,
|
||||
gpu.sram_threshold_exceeded,
|
||||
gpu.sram_l2, gpu.sram_sm, gpu.sram_microcontroller, gpu.sram_pcie, gpu.sram_other,
|
||||
gpu.dram_corr_volatile, gpu.dram_uncorr_volatile, gpu.dram_corr_aggregate, gpu.dram_uncorr_aggregate,
|
||||
gpu.hw_slowdown, gpu.hw_thermal_slowdown, gpu.hw_power_brake_slowdown,
|
||||
))
|
||||
|
||||
# XID 이벤트 저장
|
||||
for xid in result.xid_events:
|
||||
conn.execute("""
|
||||
INSERT INTO analysis_xid
|
||||
(analysis_id, timestamp, xid_code, xid_name, severity,
|
||||
pci_address, gpu_name, gpu_serial, description, action, raw_line)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?)
|
||||
""", (
|
||||
analysis_id, xid.timestamp, xid.xid_code, xid.name, xid.severity,
|
||||
xid.pci_address, xid.gpu_name, xid.gpu_serial, xid.description, xid.action, xid.raw_line,
|
||||
))
|
||||
|
||||
# 커널 이벤트 저장
|
||||
for ke in result.kernel_events:
|
||||
conn.execute("""
|
||||
INSERT INTO analysis_kernel
|
||||
(analysis_id, timestamp, level, message, raw_line)
|
||||
VALUES (?,?,?,?,?)
|
||||
""", (
|
||||
analysis_id, ke.timestamp, ke.level, ke.message, ke.raw_line,
|
||||
))
|
||||
|
||||
# RMA 판정 결과 저장
|
||||
for v in result.rma_verdicts:
|
||||
conn.execute("""
|
||||
INSERT INTO analysis_rma
|
||||
(analysis_id, gpu_index, gpu_name, serial_number,
|
||||
qualifies_for_rma, reasons, warnings,
|
||||
row_remap_details, sram_details)
|
||||
VALUES (?,?,?,?,?,?,?,?,?)
|
||||
""", (
|
||||
analysis_id, v.gpu_index, v.gpu_name, v.serial_number,
|
||||
1 if v.qualifies_for_rma else 0,
|
||||
json.dumps(v.reasons, ensure_ascii=False),
|
||||
json.dumps(v.warnings, ensure_ascii=False),
|
||||
json.dumps(v.row_remap_details, ensure_ascii=False),
|
||||
json.dumps(v.sram_details, ensure_ascii=False),
|
||||
))
|
||||
|
||||
return analysis_id
|
||||
|
||||
|
||||
def update_memo(analysis_id: int, memo: str):
|
||||
"""분석 건에 메모 추가/수정"""
|
||||
with get_conn() as conn:
|
||||
conn.execute("UPDATE analysis SET memo = ? WHERE id = ?", (memo, analysis_id))
|
||||
|
||||
|
||||
def delete_analysis(analysis_id: int):
|
||||
"""분석 건 삭제 (CASCADE로 하위 테이블도 삭제)"""
|
||||
with get_conn() as conn:
|
||||
conn.execute("DELETE FROM analysis WHERE id = ?", (analysis_id,))
|
||||
|
||||
|
||||
def get_analysis_list(limit=100, status_filter=None, search=None):
|
||||
"""분석 이력 목록 조회"""
|
||||
with get_conn() as conn:
|
||||
query = "SELECT * FROM analysis WHERE 1=1"
|
||||
params = []
|
||||
if status_filter and status_filter != "전체":
|
||||
query += " AND overall_status = ?"
|
||||
params.append(status_filter)
|
||||
if search:
|
||||
query += " AND (filename LIKE ? OR memo LIKE ? OR driver_version LIKE ?)"
|
||||
params.extend([f"%{search}%", f"%{search}%", f"%{search}%"])
|
||||
query += " ORDER BY created_at DESC LIMIT ?"
|
||||
params.append(limit)
|
||||
return [dict(row) for row in conn.execute(query, params).fetchall()]
|
||||
|
||||
|
||||
def get_analysis_detail(analysis_id: int):
|
||||
"""분석 건 상세 조회 (GPU, XID, 커널 이벤트 포함)"""
|
||||
with get_conn() as conn:
|
||||
analysis = conn.execute("SELECT * FROM analysis WHERE id = ?", (analysis_id,)).fetchone()
|
||||
if not analysis:
|
||||
return None
|
||||
result = dict(analysis)
|
||||
result["gpus"] = [dict(r) for r in conn.execute(
|
||||
"SELECT * FROM analysis_gpu WHERE analysis_id = ? ORDER BY gpu_index", (analysis_id,)
|
||||
).fetchall()]
|
||||
result["xid_events"] = [dict(r) for r in conn.execute(
|
||||
"SELECT * FROM analysis_xid WHERE analysis_id = ? ORDER BY id", (analysis_id,)
|
||||
).fetchall()]
|
||||
result["kernel_events"] = [dict(r) for r in conn.execute(
|
||||
"SELECT * FROM analysis_kernel WHERE analysis_id = ? ORDER BY id", (analysis_id,)
|
||||
).fetchall()]
|
||||
rma_rows = conn.execute(
|
||||
"SELECT * FROM analysis_rma WHERE analysis_id = ? ORDER BY gpu_index", (analysis_id,)
|
||||
).fetchall()
|
||||
result["rma_verdicts"] = []
|
||||
for r in rma_rows:
|
||||
d = dict(r)
|
||||
d["reasons"] = json.loads(d.get("reasons") or "[]")
|
||||
d["warnings"] = json.loads(d.get("warnings") or "[]")
|
||||
d["row_remap_details"] = json.loads(d.get("row_remap_details") or "{}")
|
||||
d["sram_details"] = json.loads(d.get("sram_details") or "{}")
|
||||
result["rma_verdicts"].append(d)
|
||||
return result
|
||||
|
||||
|
||||
def get_xid_statistics():
|
||||
"""전체 이력에서 XID 코드별 발생 통계"""
|
||||
with get_conn() as conn:
|
||||
return [dict(r) for r in conn.execute("""
|
||||
SELECT xid_code, xid_name, severity,
|
||||
COUNT(*) as total_count,
|
||||
COUNT(DISTINCT analysis_id) as affected_reports,
|
||||
MIN(a.created_at) as first_seen,
|
||||
MAX(a.created_at) as last_seen
|
||||
FROM analysis_xid x
|
||||
JOIN analysis a ON a.id = x.analysis_id
|
||||
GROUP BY xid_code
|
||||
ORDER BY total_count DESC
|
||||
""").fetchall()]
|
||||
|
||||
|
||||
def get_gpu_serial_history(serial_number: str):
|
||||
"""특정 GPU 시리얼 번호의 장애 이력 추적"""
|
||||
with get_conn() as conn:
|
||||
return [dict(r) for r in conn.execute("""
|
||||
SELECT a.id, a.created_at, a.filename, a.overall_status,
|
||||
a.total_issues, a.critical_count, a.memo,
|
||||
g.temperature, g.power_draw, g.memory_used, g.memory_total
|
||||
FROM analysis_gpu g
|
||||
JOIN analysis a ON a.id = g.analysis_id
|
||||
WHERE g.serial_number = ?
|
||||
ORDER BY a.created_at DESC
|
||||
""", (serial_number,)).fetchall()]
|
||||
|
||||
|
||||
def get_all_gpu_serials():
|
||||
"""DB에 기록된 모든 GPU 시리얼 번호 목록"""
|
||||
with get_conn() as conn:
|
||||
return [dict(r) for r in conn.execute("""
|
||||
SELECT DISTINCT serial_number, name, gpu_uuid,
|
||||
COUNT(DISTINCT analysis_id) as report_count
|
||||
FROM analysis_gpu
|
||||
WHERE serial_number != 'N/A'
|
||||
GROUP BY serial_number
|
||||
ORDER BY report_count DESC
|
||||
""").fetchall()]
|
||||
|
||||
|
||||
def get_status_summary():
|
||||
"""전체 분석 이력 상태 요약 (대시보드용)"""
|
||||
with get_conn() as conn:
|
||||
total = conn.execute("SELECT COUNT(*) as cnt FROM analysis").fetchone()["cnt"]
|
||||
by_status = {r["overall_status"]: r["cnt"] for r in conn.execute(
|
||||
"SELECT overall_status, COUNT(*) as cnt FROM analysis GROUP BY overall_status"
|
||||
).fetchall()}
|
||||
recent_critical = [dict(r) for r in conn.execute("""
|
||||
SELECT id, created_at, filename, critical_count, memo
|
||||
FROM analysis WHERE overall_status = 'critical'
|
||||
ORDER BY created_at DESC LIMIT 5
|
||||
""").fetchall()]
|
||||
return {
|
||||
"total_reports": total,
|
||||
"by_status": by_status,
|
||||
"recent_critical": recent_critical,
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
version: "3.8"
|
||||
|
||||
services:
|
||||
nv-log-analyzer:
|
||||
build: .
|
||||
container_name: nv-log-analyzer
|
||||
ports:
|
||||
- "8501:8501"
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
environment:
|
||||
- STREAMLIT_SERVER_MAX_UPLOAD_SIZE=500
|
||||
+997
@@ -0,0 +1,997 @@
|
||||
"""
|
||||
NV-Log Analyzer - 핵심 분석 엔진
|
||||
nvidia-bug-report.log.gz 파일을 파싱하여 XID 에러, GPU 상태, 커널 로그를 분석합니다.
|
||||
성능 최적화: 모든 파싱을 라인 단위로 처리하여 대용량 파일에서도 빠르게 동작합니다.
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import json
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# XID 데이터베이스 로드
|
||||
DATA_DIR = Path(__file__).parent / "data"
|
||||
with open(DATA_DIR / "xid_database.json", "r", encoding="utf-8") as f:
|
||||
XID_DB = json.load(f)
|
||||
|
||||
|
||||
# --- 데이터 클래스 ---
|
||||
|
||||
@dataclass
|
||||
class XidEvent:
|
||||
timestamp: str
|
||||
xid_code: int
|
||||
pci_address: str
|
||||
raw_line: str
|
||||
name: str = ""
|
||||
severity: str = "info"
|
||||
description: str = ""
|
||||
action: str = ""
|
||||
gpu_index: int = -1
|
||||
gpu_name: str = ""
|
||||
gpu_serial: str = ""
|
||||
|
||||
def __post_init__(self):
|
||||
xid_info = XID_DB["xid_errors"].get(str(self.xid_code))
|
||||
if xid_info:
|
||||
self.name = xid_info["name"]
|
||||
self.severity = xid_info["severity"]
|
||||
self.description = xid_info["description"]
|
||||
self.action = xid_info["action"]
|
||||
else:
|
||||
self.name = f"Unknown XID {self.xid_code}"
|
||||
self.severity = "warning"
|
||||
self.description = f"XID {self.xid_code}에 대한 정보가 데이터베이스에 없습니다."
|
||||
self.action = "NVIDIA 공식 문서에서 해당 XID 코드를 확인하세요."
|
||||
|
||||
|
||||
@dataclass
|
||||
class GpuInfo:
|
||||
index: int = 0
|
||||
name: str = "N/A"
|
||||
product_brand: str = "N/A"
|
||||
product_architecture: str = "N/A"
|
||||
driver_version: str = "N/A"
|
||||
cuda_version: str = "N/A"
|
||||
vbios_version: str = "N/A"
|
||||
inforom_version: str = "N/A"
|
||||
gpu_uuid: str = "N/A"
|
||||
temperature: int = 0
|
||||
temperature_limit: str = "N/A"
|
||||
power_draw: float = 0.0
|
||||
power_limit: float = 0.0
|
||||
power_state: str = "N/A"
|
||||
memory_used: int = 0
|
||||
memory_total: int = 0
|
||||
memory_free: int = 0
|
||||
bar1_total: int = 0
|
||||
bar1_used: int = 0
|
||||
gpu_utilization: int = 0
|
||||
memory_utilization: int = 0
|
||||
ecc_mode: str = "N/A"
|
||||
ecc_single_bit: int = 0
|
||||
ecc_double_bit: int = 0
|
||||
retired_pages_single: int = 0
|
||||
retired_pages_double: int = 0
|
||||
pci_address: str = "N/A"
|
||||
pci_link_gen_current: str = "N/A"
|
||||
pci_link_width_current: str = "N/A"
|
||||
serial_number: str = "N/A"
|
||||
fan_speed: int = 0
|
||||
persistence_mode: str = "N/A"
|
||||
compute_mode: str = "N/A"
|
||||
mig_mode: str = "N/A"
|
||||
clock_graphics: str = "N/A"
|
||||
clock_memory: str = "N/A"
|
||||
clock_max_graphics: str = "N/A"
|
||||
clock_max_memory: str = "N/A"
|
||||
# --- HW Slowdown ---
|
||||
hw_slowdown: str = "N/A"
|
||||
hw_thermal_slowdown: str = "N/A"
|
||||
hw_power_brake_slowdown: str = "N/A"
|
||||
# --- RMA 관련 필드 ---
|
||||
# Row Remapping
|
||||
remap_corr_error: int = 0
|
||||
remap_uncorr_error: int = 0
|
||||
remap_pending: str = "N/A"
|
||||
remap_failure: str = "N/A"
|
||||
remap_bank_max: int = 0
|
||||
remap_bank_high: int = 0
|
||||
remap_bank_partial: int = 0
|
||||
remap_bank_low: int = 0
|
||||
remap_bank_none: int = 0
|
||||
# SRAM ECC
|
||||
sram_corr_volatile: str = "N/A"
|
||||
sram_uncorr_parity_volatile: str = "N/A"
|
||||
sram_uncorr_secded_volatile: str = "N/A"
|
||||
sram_corr_aggregate: str = "N/A"
|
||||
sram_uncorr_parity_aggregate: str = "N/A"
|
||||
sram_uncorr_secded_aggregate: str = "N/A"
|
||||
sram_threshold_exceeded: str = "N/A"
|
||||
# SRAM Sources
|
||||
sram_l2: str = "N/A"
|
||||
sram_sm: str = "N/A"
|
||||
sram_microcontroller: str = "N/A"
|
||||
sram_pcie: str = "N/A"
|
||||
sram_other: str = "N/A"
|
||||
# DRAM ECC
|
||||
dram_corr_volatile: str = "N/A"
|
||||
dram_uncorr_volatile: str = "N/A"
|
||||
dram_corr_aggregate: str = "N/A"
|
||||
dram_uncorr_aggregate: str = "N/A"
|
||||
|
||||
|
||||
@dataclass
|
||||
class RmaVerdict:
|
||||
"""GPU별 RMA 판정 결과"""
|
||||
gpu_index: int = 0
|
||||
gpu_name: str = ""
|
||||
serial_number: str = ""
|
||||
qualifies_for_rma: bool = False
|
||||
reasons: list = field(default_factory=list)
|
||||
warnings: list = field(default_factory=list)
|
||||
row_remap_details: dict = field(default_factory=dict)
|
||||
sram_details: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KernelEvent:
|
||||
timestamp: str
|
||||
level: str
|
||||
message: str
|
||||
raw_line: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class AnalysisResult:
|
||||
filename: str = ""
|
||||
file_size_mb: float = 0.0
|
||||
analysis_time: str = ""
|
||||
gpus: list = field(default_factory=list)
|
||||
driver_version: str = "N/A"
|
||||
cuda_version: str = "N/A"
|
||||
xid_events: list = field(default_factory=list)
|
||||
xid_summary: dict = field(default_factory=dict)
|
||||
kernel_events: list = field(default_factory=list)
|
||||
kernel_summary: dict = field(default_factory=dict)
|
||||
ecc_info: dict = field(default_factory=dict)
|
||||
rma_verdicts: list = field(default_factory=list) # List[RmaVerdict]
|
||||
rma_count: int = 0 # RMA 대상 GPU 수
|
||||
overall_status: str = "healthy"
|
||||
total_issues: int = 0
|
||||
critical_count: int = 0
|
||||
warning_count: int = 0
|
||||
info_count: int = 0
|
||||
|
||||
|
||||
# --- 라인 단위 정규표현식 (성능 최적화) ---
|
||||
|
||||
# XID: 한 라인 안에서만 매칭
|
||||
RE_XID = re.compile(r"NVRM:\s*Xid\s*\(PCI:([^)]+)\):\s*(\d+)")
|
||||
RE_TIMESTAMP = re.compile(r"^(\w{3}\s+\d+\s+[\d:]+|\d{4}-\d{2}-\d{2}[T ][\d:]+|[\d]+\.[\d]+)")
|
||||
|
||||
# GPU 정보 (라인 단위) - nvidia-smi -q 출력의 "Key : Value" 패턴
|
||||
RE_GPU_NAME = re.compile(r"Product Name\s*:\s*(.+)", re.IGNORECASE)
|
||||
RE_GPU_BRAND = re.compile(r"Product Brand\s*:\s*(.+)", re.IGNORECASE)
|
||||
RE_GPU_ARCH = re.compile(r"Product Architecture\s*:\s*(.+)", re.IGNORECASE)
|
||||
RE_DRIVER = re.compile(r"Driver Version\s*:\s*([\d.]+)", re.IGNORECASE)
|
||||
RE_CUDA = re.compile(r"CUDA Version\s*:\s*([\d.]+)", re.IGNORECASE)
|
||||
RE_VBIOS = re.compile(r"VBIOS Version\s*:\s*(\S+)", re.IGNORECASE)
|
||||
RE_INFOROM = re.compile(r"Image Version\s*:\s*(\S+)", re.IGNORECASE)
|
||||
RE_UUID = re.compile(r"GPU UUID\s*:\s*(GPU-\S+)", re.IGNORECASE)
|
||||
RE_TEMP = re.compile(r"GPU Current Temp\s*:\s*(\d+)\s*C", re.IGNORECASE)
|
||||
RE_TEMP_LIMIT = re.compile(r"GPU T\.?Limit Temp\s*:\s*(.+)", re.IGNORECASE)
|
||||
RE_POWER_DRAW = re.compile(r"Power Draw\s*:\s*([\d.]+)\s*W", re.IGNORECASE)
|
||||
RE_POWER_LIMIT = re.compile(r"(?:Default |Enforced |Current )?Power Limit\s*:\s*([\d.]+)\s*W", re.IGNORECASE)
|
||||
RE_POWER_STATE = re.compile(r"Power State\s*:\s*(\S+)", re.IGNORECASE)
|
||||
RE_MEM_USED = re.compile(r"Used\s*:\s*(\d+)\s*MiB", re.IGNORECASE)
|
||||
RE_MEM_TOTAL = re.compile(r"Total\s*:\s*(\d+)\s*MiB", re.IGNORECASE)
|
||||
RE_MEM_FREE = re.compile(r"Free\s*:\s*(\d+)\s*MiB", re.IGNORECASE)
|
||||
RE_GPU_UTIL = re.compile(r"Gpu\s*:\s*(\d+)\s*%", re.IGNORECASE)
|
||||
RE_MEM_UTIL = re.compile(r"Memory\s*:\s*(\d+)\s*%", re.IGNORECASE)
|
||||
RE_PCI = re.compile(r"Bus Id\s*:\s*([\dA-Fa-f:.]+)", re.IGNORECASE)
|
||||
RE_PCI_GEN = re.compile(r"Current\s*:\s*Gen(\d+)", re.IGNORECASE)
|
||||
RE_PCI_WIDTH = re.compile(r"Current\s*:\s*(\d+x)", re.IGNORECASE)
|
||||
RE_SERIAL = re.compile(r"^\s*Serial Number\s*:\s*(\S+)", re.IGNORECASE)
|
||||
RE_FAN = re.compile(r"Fan Speed\s*:\s*(\d+)\s*%", re.IGNORECASE)
|
||||
RE_PERSIST = re.compile(r"Persistence Mode\s*:\s*(\w+)", re.IGNORECASE)
|
||||
RE_COMPUTE_MODE = re.compile(r"Compute Mode\s*:\s*(.+)", re.IGNORECASE)
|
||||
RE_MIG_MODE = re.compile(r"MIG Mode\s*:\s*(\w+)", re.IGNORECASE)
|
||||
RE_ECC_MODE = re.compile(r"ECC Mode[\s\S]*?Current\s*:\s*(\w+)", re.IGNORECASE)
|
||||
RE_CLOCK_GRAPHICS = re.compile(r"Graphics\s*:\s*(\d+)\s*MHz", re.IGNORECASE)
|
||||
RE_CLOCK_MEMORY = re.compile(r"SM\s*:\s*(\d+)\s*MHz|Memory\s*:\s*(\d+)\s*MHz", re.IGNORECASE)
|
||||
RE_HW_SLOWDOWN = re.compile(r"HW Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE)
|
||||
RE_HW_THERMAL_SLOWDOWN = re.compile(r"HW Thermal Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE)
|
||||
RE_HW_POWER_BRAKE_SLOWDOWN = re.compile(r"HW Power Brake Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE)
|
||||
# GPU 섹션 헤더: "GPU 00000000:19:00.0" 또는 "GPU 0000:01:00.0" 또는 "GPU 0:"
|
||||
RE_GPU_SECTION = re.compile(r"^GPU\s+([\dA-Fa-f]{4,8}:[\dA-Fa-f]{2}:[\dA-Fa-f]{2}\.\d)\s*$", re.IGNORECASE)
|
||||
RE_GPU_SECTION_IDX = re.compile(r"^GPU\s+(\d+)\s*:\s*$", re.IGNORECASE)
|
||||
|
||||
# nvidia-smi 테이블 행
|
||||
RE_SMI_ROW = re.compile(
|
||||
r"\|\s*(\d+)\s+(.+?)\s+\w+\s*\|.*?(\d+)C.*?([\d.]+)W\s*/\s*([\d.]+)W.*?(\d+)MiB\s*/\s*(\d+)MiB.*?(\d+)%"
|
||||
)
|
||||
|
||||
# ECC (라인 단위)
|
||||
RE_SINGLE_BIT = re.compile(r"Single Bit", re.IGNORECASE)
|
||||
RE_DOUBLE_BIT = re.compile(r"Double Bit", re.IGNORECASE)
|
||||
RE_AGGREGATE = re.compile(r"Aggregate\s*:\s*(\d+)", re.IGNORECASE)
|
||||
RE_RETIRED = re.compile(r"Retired pages\s*:\s*(\d+)", re.IGNORECASE)
|
||||
RE_PENDING = re.compile(r"Pending\s*.*:\s*Yes", re.IGNORECASE)
|
||||
|
||||
# --- RMA 관련 정규표현식 ---
|
||||
# Row Remapping
|
||||
RE_REMAP_SECTION = re.compile(r"Remapped Rows", re.IGNORECASE)
|
||||
RE_REMAP_CORR = re.compile(r"Correctable Error\s*:\s*(\d+)", re.IGNORECASE)
|
||||
RE_REMAP_UNCORR = re.compile(r"Uncorrectable Error\s*:\s*(\d+)", re.IGNORECASE)
|
||||
RE_REMAP_PENDING = re.compile(r"Pending\s*:\s*(Yes|No)", re.IGNORECASE)
|
||||
RE_REMAP_FAILURE = re.compile(r"Remapping Failure Occurred\s*:\s*(Yes|No)", re.IGNORECASE)
|
||||
RE_BANK_MAX = re.compile(r"Max\s*:\s*(\d+)\s*bank", re.IGNORECASE)
|
||||
RE_BANK_HIGH = re.compile(r"High\s*:\s*(\d+)\s*bank", re.IGNORECASE)
|
||||
RE_BANK_PARTIAL = re.compile(r"Partial\s*:\s*(\d+)\s*bank", re.IGNORECASE)
|
||||
RE_BANK_LOW = re.compile(r"Low\s*:\s*(\d+)\s*bank", re.IGNORECASE)
|
||||
RE_BANK_NONE = re.compile(r"None\s*:\s*(\d+)\s*bank", re.IGNORECASE)
|
||||
# SRAM ECC
|
||||
RE_SRAM_CORR = re.compile(r"SRAM Correctable\s*:\s*(\S+)", re.IGNORECASE)
|
||||
RE_SRAM_UNCORR_PARITY = re.compile(r"SRAM Uncorrectable Parity\s*:\s*(\S+)", re.IGNORECASE)
|
||||
RE_SRAM_UNCORR_SECDED = re.compile(r"SRAM Uncorrectable SEC-?DED\s*:\s*(\S+)", re.IGNORECASE)
|
||||
RE_SRAM_THRESHOLD = re.compile(r"SRAM Threshold Exceeded\s*:\s*(\S+)", re.IGNORECASE)
|
||||
# SRAM Sources
|
||||
RE_SRAM_L2 = re.compile(r"SRAM L2\s*:\s*(\S+)", re.IGNORECASE)
|
||||
RE_SRAM_SM = re.compile(r"SRAM SM\s*:\s*(\S+)", re.IGNORECASE)
|
||||
RE_SRAM_MC = re.compile(r"SRAM Microcontroller\s*:\s*(\S+)", re.IGNORECASE)
|
||||
RE_SRAM_PCIE = re.compile(r"SRAM PCIE\s*:\s*(\S+)", re.IGNORECASE)
|
||||
RE_SRAM_OTHER = re.compile(r"SRAM Other\s*:\s*(\S+)", re.IGNORECASE)
|
||||
# DRAM ECC
|
||||
RE_DRAM_CORR = re.compile(r"DRAM Correctable\s*:\s*(\S+)", re.IGNORECASE)
|
||||
RE_DRAM_UNCORR = re.compile(r"DRAM Uncorrectable\s*:\s*(\S+)", re.IGNORECASE)
|
||||
# ECC Section context
|
||||
RE_ECC_SECTION = re.compile(r"ECC Errors", re.IGNORECASE)
|
||||
RE_VOLATILE = re.compile(r"^\s*Volatile\s*$", re.IGNORECASE)
|
||||
RE_AGGREGATE_SECTION = re.compile(r"^\s*Aggregate\s*$", re.IGNORECASE)
|
||||
RE_AGG_SRAM_SOURCES = re.compile(r"Aggregate Uncorrectable SRAM Sources", re.IGNORECASE)
|
||||
|
||||
# 커널 이벤트 (라인 단위 키워드 매칭)
|
||||
KERNEL_CRITICAL_KW = re.compile(
|
||||
r"(kernel panic|hardware error|machine check|fatal|BUG:|RIP:)", re.IGNORECASE
|
||||
)
|
||||
KERNEL_ERROR_KW = re.compile(
|
||||
r"(error|failed|failure|segfault|cannot|unable to)", re.IGNORECASE
|
||||
)
|
||||
KERNEL_OOM_KW = re.compile(
|
||||
r"(Out of memory|oom-killer|invoked oom|Killed process)", re.IGNORECASE
|
||||
)
|
||||
KERNEL_PCIE_KW = re.compile(
|
||||
r"(AER|correctable error|uncorrectable error)", re.IGNORECASE
|
||||
)
|
||||
KERNEL_SKIP_KW = re.compile(r"(audit|systemd|session|polkit|dbus)", re.IGNORECASE)
|
||||
|
||||
# 섹션 구분: 언더스코어(_) 또는 대시(-)가 양옆에 있고 대괄호([, ])가 있을 수도 없을 수도 있는 구분선 매칭
|
||||
# nvidia-bug-report.log 내 dmesg 등 다양한 로그 섹션 구분선을 감지합니다.
|
||||
RE_SECTION = re.compile(r"^[_-]+\s*\[?(.+?)\]?\s*[_-]+$")
|
||||
|
||||
|
||||
# --- 파서 함수들 ---
|
||||
|
||||
_RE_PCI_NORMALIZE = re.compile(
|
||||
r"^([\dA-Fa-f]+):([\dA-Fa-f]{1,2}):([\dA-Fa-f]{1,2})(?:\.\d)?$"
|
||||
)
|
||||
_RE_PCI_NO_DOMAIN = re.compile(
|
||||
r"^([\dA-Fa-f]{1,2}):([\dA-Fa-f]{1,2})(?:\.\d)?$"
|
||||
)
|
||||
|
||||
def _norm_pci(addr: str) -> str:
|
||||
"""
|
||||
PCI 주소를 'DOMAIN:BUS:DEVICE' 형태로 정규화.
|
||||
도메인은 정수값으로, bus/device는 소문자 2자리 hex로 통일.
|
||||
- '0000:19:00' → '0:19:00'
|
||||
- '00000000:19:00.0' → '0:19:00'
|
||||
- '0001:3b:00.0' → '1:3b:00' (다른 도메인 → 다른 키)
|
||||
- '19:00.0' → '0:19:00' (도메인 생략 → 0으로 가정)
|
||||
매칭 실패 시 원문 소문자 반환.
|
||||
"""
|
||||
s = addr.strip()
|
||||
m = _RE_PCI_NORMALIZE.match(s)
|
||||
if m:
|
||||
domain = int(m.group(1), 16)
|
||||
bus = m.group(2).zfill(2).lower()
|
||||
device = m.group(3).zfill(2).lower()
|
||||
return f"{domain}:{bus}:{device}"
|
||||
m2 = _RE_PCI_NO_DOMAIN.match(s)
|
||||
if m2:
|
||||
bus = m2.group(1).zfill(2).lower()
|
||||
device = m2.group(2).zfill(2).lower()
|
||||
return f"0:{bus}:{device}"
|
||||
return s.lower()
|
||||
|
||||
|
||||
def read_log_file(file_content: bytes, filename: str = "") -> str:
|
||||
"""gz 또는 텍스트 파일을 읽어 문자열로 반환"""
|
||||
if filename.endswith(".gz") or file_content[:2] == b'\x1f\x8b':
|
||||
try:
|
||||
return gzip.decompress(file_content).decode("utf-8", errors="replace")
|
||||
except gzip.BadGzipFile:
|
||||
return file_content.decode("utf-8", errors="replace")
|
||||
return file_content.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def analyze_log(file_content: bytes, filename: str = "uploaded_file") -> AnalysisResult:
|
||||
"""메인 분석 함수 - 라인 단위 싱글 패스 처리"""
|
||||
result = AnalysisResult(
|
||||
filename=filename,
|
||||
file_size_mb=round(len(file_content) / (1024 * 1024), 2),
|
||||
analysis_time=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
)
|
||||
|
||||
log_text = read_log_file(file_content, filename)
|
||||
lines = log_text.splitlines()
|
||||
|
||||
# --- 상태 변수들 ---
|
||||
current_section = "header"
|
||||
driver_version = None
|
||||
cuda_version = None
|
||||
|
||||
# GPU 파싱 상태
|
||||
gpu_list = []
|
||||
current_gpu = None
|
||||
smi_gpus_found = False
|
||||
|
||||
# NVSMI LOG 상세 섹션의 PCI → Serial 별도 수집 (smi_gpus_found 여부 무관)
|
||||
pci_serial_map: dict = {} # { "00000000:19:00.0" : "serial_string" }
|
||||
_nvsmi_cur_pci: str = "" # 현재 GPU PCI (NVSMI 상세 파싱용)
|
||||
|
||||
# XID 이벤트
|
||||
xid_events = []
|
||||
|
||||
# 커널 이벤트 (dmesg 섹션만)
|
||||
kernel_events = []
|
||||
seen_kernel = set()
|
||||
in_dmesg = False
|
||||
|
||||
# RMA/ECC 파싱 상태 머신
|
||||
in_ecc_section = False
|
||||
ecc_subsection = None # "volatile", "aggregate", "agg_sources"
|
||||
in_remap_section = False
|
||||
in_bank_histogram = False
|
||||
|
||||
# ECC 상태
|
||||
ecc_single_total = 0
|
||||
ecc_double_total = 0
|
||||
ecc_retired = 0
|
||||
ecc_pending = False
|
||||
ecc_context = None # "single" or "double" 마지막으로 본 ECC 유형
|
||||
|
||||
# --- 라인 단위 파싱 (싱글 패스) ---
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
|
||||
# 섹션 감지
|
||||
sec_match = RE_SECTION.match(stripped)
|
||||
if sec_match:
|
||||
current_section = sec_match.group(1).strip().lower()
|
||||
in_dmesg = any(kw in current_section for kw in ["dmesg", "kernel", "messages"])
|
||||
continue
|
||||
|
||||
# --- NVSMI LOG: GPU PCI 헤더 및 Serial Number 항상 수집 ---
|
||||
gpu_sec_always = RE_GPU_SECTION.match(stripped)
|
||||
if gpu_sec_always:
|
||||
_nvsmi_cur_pci = _norm_pci(gpu_sec_always.group(1))
|
||||
elif _nvsmi_cur_pci:
|
||||
serial_m = RE_SERIAL.search(stripped)
|
||||
if serial_m and "chassis" not in stripped.lower():
|
||||
val = serial_m.group(1).strip()
|
||||
if val and val != "N/A":
|
||||
pci_serial_map[_nvsmi_cur_pci] = val
|
||||
|
||||
# --- XID 에러 (모든 섹션) ---
|
||||
xid_m = RE_XID.search(stripped)
|
||||
if xid_m:
|
||||
ts_m = RE_TIMESTAMP.match(stripped)
|
||||
timestamp = ts_m.group(1) if ts_m else ""
|
||||
xid_events.append(XidEvent(
|
||||
timestamp=timestamp,
|
||||
xid_code=int(xid_m.group(2)),
|
||||
pci_address=xid_m.group(1).strip(),
|
||||
raw_line=stripped[:500],
|
||||
))
|
||||
continue
|
||||
|
||||
# --- 드라이버/CUDA 버전 (최초 발견만) ---
|
||||
if not driver_version:
|
||||
dm = RE_DRIVER.search(stripped)
|
||||
if dm:
|
||||
driver_version = dm.group(1)
|
||||
if not cuda_version:
|
||||
cm = RE_CUDA.search(stripped)
|
||||
if cm:
|
||||
cuda_version = cm.group(1)
|
||||
|
||||
# --- nvidia-smi 테이블 행 ---
|
||||
smi_m = RE_SMI_ROW.search(stripped)
|
||||
if smi_m:
|
||||
smi_gpus_found = True
|
||||
gpu_list.append(GpuInfo(
|
||||
index=int(smi_m.group(1)),
|
||||
name=smi_m.group(2).strip(),
|
||||
temperature=int(smi_m.group(3)),
|
||||
power_draw=float(smi_m.group(4)),
|
||||
power_limit=float(smi_m.group(5)),
|
||||
memory_used=int(smi_m.group(6)),
|
||||
memory_total=int(smi_m.group(7)),
|
||||
gpu_utilization=int(smi_m.group(8)),
|
||||
))
|
||||
continue
|
||||
|
||||
# --- nvidia-smi -q 상세 형식 (GPU 섹션 기반) ---
|
||||
if not smi_gpus_found:
|
||||
# GPU 섹션 헤더 감지: "GPU 00000000:19:00.0" 또는 "GPU 0:"
|
||||
gpu_sec_m = RE_GPU_SECTION.match(stripped) or RE_GPU_SECTION_IDX.match(stripped)
|
||||
if gpu_sec_m:
|
||||
if current_gpu and current_gpu.name != "N/A":
|
||||
gpu_list.append(current_gpu)
|
||||
current_gpu = GpuInfo(index=len(gpu_list))
|
||||
# PCI 주소가 헤더에 포함된 경우
|
||||
addr = gpu_sec_m.group(1)
|
||||
if ":" in addr and "." in addr:
|
||||
current_gpu.pci_address = addr
|
||||
continue
|
||||
|
||||
if current_gpu is not None:
|
||||
# 제품 정보
|
||||
m = RE_GPU_NAME.search(stripped)
|
||||
if m:
|
||||
current_gpu.name = m.group(1).strip()
|
||||
continue
|
||||
m = RE_GPU_BRAND.search(stripped)
|
||||
if m:
|
||||
current_gpu.product_brand = m.group(1).strip()
|
||||
continue
|
||||
m = RE_GPU_ARCH.search(stripped)
|
||||
if m:
|
||||
current_gpu.product_architecture = m.group(1).strip()
|
||||
continue
|
||||
m = RE_UUID.search(stripped)
|
||||
if m:
|
||||
current_gpu.gpu_uuid = m.group(1).strip()
|
||||
continue
|
||||
m = RE_SERIAL.search(stripped)
|
||||
if m and "chassis" not in stripped.lower():
|
||||
val = m.group(1).strip()
|
||||
if val != "N/A":
|
||||
current_gpu.serial_number = val
|
||||
continue
|
||||
m = RE_VBIOS.search(stripped)
|
||||
if m:
|
||||
current_gpu.vbios_version = m.group(1).strip()
|
||||
continue
|
||||
m = RE_INFOROM.search(stripped)
|
||||
if m:
|
||||
current_gpu.inforom_version = m.group(1).strip()
|
||||
continue
|
||||
|
||||
# 온도
|
||||
m = RE_TEMP.search(stripped)
|
||||
if m:
|
||||
current_gpu.temperature = int(m.group(1))
|
||||
continue
|
||||
m = RE_TEMP_LIMIT.search(stripped)
|
||||
if m:
|
||||
current_gpu.temperature_limit = m.group(1).strip()
|
||||
continue
|
||||
|
||||
# 전력
|
||||
m = RE_POWER_DRAW.search(stripped)
|
||||
if m:
|
||||
current_gpu.power_draw = float(m.group(1))
|
||||
continue
|
||||
m = RE_POWER_LIMIT.search(stripped)
|
||||
if m:
|
||||
current_gpu.power_limit = float(m.group(1))
|
||||
continue
|
||||
m = RE_POWER_STATE.search(stripped)
|
||||
if m:
|
||||
current_gpu.power_state = m.group(1).strip()
|
||||
continue
|
||||
|
||||
# 메모리 (FB Memory Usage 값만 유지, 이후 Conf Compute 등에 덮어쓰기 방지)
|
||||
m = RE_MEM_TOTAL.search(stripped)
|
||||
if m:
|
||||
val = int(m.group(1))
|
||||
if current_gpu.memory_total == 0 and val > 0:
|
||||
current_gpu.memory_total = val
|
||||
continue
|
||||
m = RE_MEM_USED.search(stripped)
|
||||
if m:
|
||||
val = int(m.group(1))
|
||||
if current_gpu.memory_used == 0:
|
||||
current_gpu.memory_used = val
|
||||
continue
|
||||
m = RE_MEM_FREE.search(stripped)
|
||||
if m:
|
||||
val = int(m.group(1))
|
||||
if current_gpu.memory_free == 0 and val > 0:
|
||||
current_gpu.memory_free = val
|
||||
continue
|
||||
|
||||
# 사용률
|
||||
m = RE_GPU_UTIL.search(stripped)
|
||||
if m:
|
||||
current_gpu.gpu_utilization = int(m.group(1))
|
||||
continue
|
||||
m = RE_MEM_UTIL.search(stripped)
|
||||
if m:
|
||||
current_gpu.memory_utilization = int(m.group(1))
|
||||
continue
|
||||
|
||||
# PCI
|
||||
m = RE_PCI.search(stripped)
|
||||
if m:
|
||||
current_gpu.pci_address = m.group(1).strip()
|
||||
continue
|
||||
m = RE_PCI_GEN.search(stripped)
|
||||
if m:
|
||||
current_gpu.pci_link_gen_current = f"Gen{m.group(1)}"
|
||||
continue
|
||||
m = RE_PCI_WIDTH.search(stripped)
|
||||
if m:
|
||||
current_gpu.pci_link_width_current = m.group(1)
|
||||
continue
|
||||
|
||||
# 기타
|
||||
m = RE_FAN.search(stripped)
|
||||
if m:
|
||||
current_gpu.fan_speed = int(m.group(1))
|
||||
continue
|
||||
m = RE_PERSIST.search(stripped)
|
||||
if m:
|
||||
current_gpu.persistence_mode = m.group(1).strip()
|
||||
continue
|
||||
m = RE_COMPUTE_MODE.search(stripped)
|
||||
if m:
|
||||
current_gpu.compute_mode = m.group(1).strip()
|
||||
continue
|
||||
m = RE_MIG_MODE.search(stripped)
|
||||
if m:
|
||||
current_gpu.mig_mode = m.group(1).strip()
|
||||
continue
|
||||
m = RE_ECC_MODE.search(stripped)
|
||||
if m:
|
||||
current_gpu.ecc_mode = m.group(1).strip()
|
||||
continue
|
||||
m = RE_CLOCK_GRAPHICS.search(stripped)
|
||||
if m:
|
||||
current_gpu.clock_graphics = f"{m.group(1)} MHz"
|
||||
continue
|
||||
|
||||
# HW Slowdown (순서 중요: 더 구체적인 패턴 먼저)
|
||||
m = RE_HW_THERMAL_SLOWDOWN.search(stripped)
|
||||
if m:
|
||||
current_gpu.hw_thermal_slowdown = m.group(1)
|
||||
continue
|
||||
m = RE_HW_POWER_BRAKE_SLOWDOWN.search(stripped)
|
||||
if m:
|
||||
current_gpu.hw_power_brake_slowdown = m.group(1)
|
||||
continue
|
||||
m = RE_HW_SLOWDOWN.search(stripped)
|
||||
if m:
|
||||
current_gpu.hw_slowdown = m.group(1)
|
||||
continue
|
||||
|
||||
# --- RMA: ECC 섹션 파싱 (GPU 내부) ---
|
||||
if RE_ECC_SECTION.match(stripped):
|
||||
in_ecc_section = True
|
||||
ecc_subsection = None
|
||||
continue
|
||||
if in_ecc_section:
|
||||
if RE_VOLATILE.match(stripped):
|
||||
ecc_subsection = "volatile"
|
||||
continue
|
||||
if RE_AGGREGATE_SECTION.match(stripped):
|
||||
ecc_subsection = "aggregate"
|
||||
continue
|
||||
if RE_AGG_SRAM_SOURCES.search(stripped):
|
||||
ecc_subsection = "agg_sources"
|
||||
continue
|
||||
|
||||
if ecc_subsection == "volatile":
|
||||
m = RE_SRAM_CORR.search(stripped)
|
||||
if m: current_gpu.sram_corr_volatile = m.group(1); continue
|
||||
m = RE_SRAM_UNCORR_PARITY.search(stripped)
|
||||
if m: current_gpu.sram_uncorr_parity_volatile = m.group(1); continue
|
||||
m = RE_SRAM_UNCORR_SECDED.search(stripped)
|
||||
if m: current_gpu.sram_uncorr_secded_volatile = m.group(1); continue
|
||||
m = RE_DRAM_CORR.search(stripped)
|
||||
if m: current_gpu.dram_corr_volatile = m.group(1); continue
|
||||
m = RE_DRAM_UNCORR.search(stripped)
|
||||
if m: current_gpu.dram_uncorr_volatile = m.group(1); continue
|
||||
|
||||
elif ecc_subsection == "aggregate":
|
||||
m = RE_SRAM_CORR.search(stripped)
|
||||
if m: current_gpu.sram_corr_aggregate = m.group(1); continue
|
||||
m = RE_SRAM_UNCORR_PARITY.search(stripped)
|
||||
if m: current_gpu.sram_uncorr_parity_aggregate = m.group(1); continue
|
||||
m = RE_SRAM_UNCORR_SECDED.search(stripped)
|
||||
if m: current_gpu.sram_uncorr_secded_aggregate = m.group(1); continue
|
||||
m = RE_DRAM_CORR.search(stripped)
|
||||
if m: current_gpu.dram_corr_aggregate = m.group(1); continue
|
||||
m = RE_DRAM_UNCORR.search(stripped)
|
||||
if m: current_gpu.dram_uncorr_aggregate = m.group(1); continue
|
||||
m = RE_SRAM_THRESHOLD.search(stripped)
|
||||
if m: current_gpu.sram_threshold_exceeded = m.group(1); continue
|
||||
|
||||
elif ecc_subsection == "agg_sources":
|
||||
m = RE_SRAM_L2.search(stripped)
|
||||
if m: current_gpu.sram_l2 = m.group(1); continue
|
||||
m = RE_SRAM_SM.search(stripped)
|
||||
if m: current_gpu.sram_sm = m.group(1); continue
|
||||
m = RE_SRAM_MC.search(stripped)
|
||||
if m: current_gpu.sram_microcontroller = m.group(1); continue
|
||||
m = RE_SRAM_PCIE.search(stripped)
|
||||
if m: current_gpu.sram_pcie = m.group(1); continue
|
||||
m = RE_SRAM_OTHER.search(stripped)
|
||||
if m: current_gpu.sram_other = m.group(1); continue
|
||||
|
||||
# --- RMA: Remapped Rows 섹션 파싱 (GPU 내부) ---
|
||||
if RE_REMAP_SECTION.match(stripped):
|
||||
in_remap_section = True
|
||||
in_ecc_section = False
|
||||
in_bank_histogram = False
|
||||
continue
|
||||
if in_remap_section:
|
||||
m = RE_REMAP_CORR.search(stripped)
|
||||
if m and "remap" not in stripped.lower():
|
||||
current_gpu.remap_corr_error = int(m.group(1)); continue
|
||||
m = RE_REMAP_UNCORR.search(stripped)
|
||||
if m:
|
||||
current_gpu.remap_uncorr_error = int(m.group(1)); continue
|
||||
m = RE_REMAP_FAILURE.search(stripped)
|
||||
if m:
|
||||
current_gpu.remap_failure = m.group(1); continue
|
||||
m = RE_REMAP_PENDING.search(stripped)
|
||||
if m and "page" not in stripped.lower():
|
||||
current_gpu.remap_pending = m.group(1); continue
|
||||
if "Bank Remap Availability" in stripped:
|
||||
in_bank_histogram = True; continue
|
||||
if in_bank_histogram:
|
||||
m = RE_BANK_MAX.search(stripped)
|
||||
if m: current_gpu.remap_bank_max = int(m.group(1)); continue
|
||||
m = RE_BANK_HIGH.search(stripped)
|
||||
if m: current_gpu.remap_bank_high = int(m.group(1)); continue
|
||||
m = RE_BANK_PARTIAL.search(stripped)
|
||||
if m: current_gpu.remap_bank_partial = int(m.group(1)); continue
|
||||
m = RE_BANK_LOW.search(stripped)
|
||||
if m: current_gpu.remap_bank_low = int(m.group(1)); continue
|
||||
m = RE_BANK_NONE.search(stripped)
|
||||
if m:
|
||||
current_gpu.remap_bank_none = int(m.group(1))
|
||||
in_remap_section = False
|
||||
in_bank_histogram = False
|
||||
continue
|
||||
|
||||
# --- ECC 에러 전체 집계 (라인 단위 상태 머신) ---
|
||||
if RE_SINGLE_BIT.search(stripped) and "sram" not in stripped.lower():
|
||||
ecc_context = "single"
|
||||
continue
|
||||
if RE_DOUBLE_BIT.search(stripped) and "sram" not in stripped.lower():
|
||||
ecc_context = "double"
|
||||
continue
|
||||
if ecc_context:
|
||||
agg_m = RE_AGGREGATE.search(stripped)
|
||||
if agg_m:
|
||||
val = int(agg_m.group(1))
|
||||
if ecc_context == "single":
|
||||
ecc_single_total = max(ecc_single_total, val)
|
||||
else:
|
||||
ecc_double_total = max(ecc_double_total, val)
|
||||
ecc_context = None
|
||||
continue
|
||||
|
||||
ret_m = RE_RETIRED.search(stripped)
|
||||
if ret_m:
|
||||
ecc_retired = max(ecc_retired, int(ret_m.group(1)))
|
||||
continue
|
||||
if RE_PENDING.search(stripped) and "remap" not in stripped.lower():
|
||||
ecc_pending = True
|
||||
continue
|
||||
|
||||
# --- 커널 이벤트 (dmesg 섹션) ---
|
||||
if in_dmesg and stripped not in seen_kernel:
|
||||
if KERNEL_SKIP_KW.search(stripped):
|
||||
continue
|
||||
|
||||
level = None
|
||||
keyword = ""
|
||||
# 우선순위: oom > critical > pcie > error
|
||||
m = KERNEL_OOM_KW.search(stripped)
|
||||
if m:
|
||||
level, keyword = "oom", m.group(1)
|
||||
if not level:
|
||||
m = KERNEL_CRITICAL_KW.search(stripped)
|
||||
if m:
|
||||
level, keyword = "critical", m.group(1)
|
||||
if not level:
|
||||
m = KERNEL_PCIE_KW.search(stripped)
|
||||
if m:
|
||||
level, keyword = "pcie_error", m.group(1)
|
||||
if not level:
|
||||
m = KERNEL_ERROR_KW.search(stripped)
|
||||
if m:
|
||||
level, keyword = "error", m.group(1)
|
||||
|
||||
if level:
|
||||
seen_kernel.add(stripped)
|
||||
ts_m = RE_TIMESTAMP.match(stripped)
|
||||
kernel_events.append(KernelEvent(
|
||||
timestamp=ts_m.group(1) if ts_m else "",
|
||||
level=level,
|
||||
message=keyword,
|
||||
raw_line=stripped[:500],
|
||||
))
|
||||
|
||||
# --- 마지막 GPU 저장 ---
|
||||
if current_gpu and current_gpu.name != "N/A" and not smi_gpus_found:
|
||||
gpu_list.append(current_gpu)
|
||||
|
||||
# --- 결과 조립 ---
|
||||
result.driver_version = driver_version or "N/A"
|
||||
result.cuda_version = cuda_version or "N/A"
|
||||
|
||||
for gpu in gpu_list:
|
||||
gpu.driver_version = result.driver_version
|
||||
gpu.cuda_version = result.cuda_version
|
||||
result.gpus = gpu_list
|
||||
|
||||
# gpu_list 시리얼 보완: NVSMI LOG에서 수집한 pci_serial_map 활용
|
||||
for gpu in gpu_list:
|
||||
if (not gpu.serial_number or gpu.serial_number == "N/A") and gpu.pci_address:
|
||||
s = pci_serial_map.get(_norm_pci(gpu.pci_address))
|
||||
if s:
|
||||
gpu.serial_number = s
|
||||
|
||||
# PCI 주소 → GPU 매핑 (정규화 키 사용)
|
||||
pci_to_gpu = {}
|
||||
for gpu in gpu_list:
|
||||
if gpu.pci_address and gpu.pci_address != "N/A":
|
||||
pci_to_gpu[_norm_pci(gpu.pci_address)] = gpu
|
||||
for event in xid_events:
|
||||
key = _norm_pci(event.pci_address)
|
||||
matched = pci_to_gpu.get(key)
|
||||
if matched:
|
||||
event.gpu_index = matched.index
|
||||
event.gpu_name = matched.name
|
||||
event.gpu_serial = matched.serial_number or pci_serial_map.get(key, "")
|
||||
else:
|
||||
# gpu_list에 없어도 pci_serial_map에서 시리얼 보완
|
||||
event.gpu_serial = pci_serial_map.get(key, "")
|
||||
|
||||
result.xid_events = xid_events
|
||||
xid_counter = Counter(e.xid_code for e in xid_events)
|
||||
result.xid_summary = {
|
||||
code: {
|
||||
"count": count,
|
||||
"severity": XID_DB["xid_errors"].get(str(code), {}).get("severity", "warning"),
|
||||
"name": XID_DB["xid_errors"].get(str(code), {}).get("name", f"Unknown XID {code}"),
|
||||
"gpus": sorted({
|
||||
f"GPU {e.gpu_index} {e.gpu_name} [S/N: {e.gpu_serial}]" if e.gpu_serial
|
||||
else (f"GPU {e.gpu_index} ({e.gpu_name})" if e.gpu_name else e.pci_address)
|
||||
for e in xid_events if e.xid_code == code
|
||||
}),
|
||||
}
|
||||
for code, count in xid_counter.most_common()
|
||||
}
|
||||
|
||||
result.kernel_events = kernel_events
|
||||
kernel_counter = Counter(e.level for e in kernel_events)
|
||||
result.kernel_summary = dict(kernel_counter)
|
||||
|
||||
result.ecc_info = {
|
||||
"single_bit_total": ecc_single_total,
|
||||
"double_bit_total": ecc_double_total,
|
||||
"retired_pages": ecc_retired,
|
||||
"pending_retirement": ecc_pending,
|
||||
}
|
||||
|
||||
# --- RMA 판정 ---
|
||||
result.rma_verdicts = [evaluate_rma(gpu) for gpu in gpu_list]
|
||||
result.rma_count = sum(1 for v in result.rma_verdicts if v.qualifies_for_rma)
|
||||
|
||||
# 상태 판정
|
||||
critical_xids = sum(1 for e in xid_events if e.severity == "critical")
|
||||
warning_xids = sum(1 for e in xid_events if e.severity == "warning")
|
||||
|
||||
result.critical_count = (
|
||||
critical_xids
|
||||
+ kernel_counter.get("critical", 0)
|
||||
+ kernel_counter.get("oom", 0)
|
||||
+ (1 if ecc_double_total > 0 else 0)
|
||||
+ result.rma_count # RMA 대상 GPU 수도 critical에 포함
|
||||
)
|
||||
result.warning_count = (
|
||||
warning_xids
|
||||
+ kernel_counter.get("error", 0)
|
||||
+ kernel_counter.get("pcie_error", 0)
|
||||
+ (1 if ecc_single_total > 10 else 0)
|
||||
)
|
||||
result.info_count = sum(1 for e in xid_events if e.severity == "info")
|
||||
result.total_issues = result.critical_count + result.warning_count
|
||||
|
||||
if result.rma_count > 0:
|
||||
result.overall_status = "critical"
|
||||
elif result.critical_count > 0:
|
||||
result.overall_status = "critical"
|
||||
elif result.warning_count > 0:
|
||||
result.overall_status = "warning"
|
||||
else:
|
||||
result.overall_status = "healthy"
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _safe_int(val: str) -> int:
|
||||
"""N/A 등 숫자가 아닌 값을 0으로 변환"""
|
||||
try:
|
||||
return int(val)
|
||||
except (ValueError, TypeError):
|
||||
return 0
|
||||
|
||||
|
||||
def evaluate_rma(gpu: GpuInfo) -> RmaVerdict:
|
||||
"""
|
||||
NVIDIA Hopper RMA 기준에 따라 GPU의 RMA 적격 여부를 판정합니다.
|
||||
|
||||
[Row-Remapping RMA 기준]
|
||||
1. Remapping Failure Occurred = Yes → 즉시 RMA
|
||||
2. 한 Bank에서 Uncorrectable Error Row 8회 이상 리매핑
|
||||
3. 총 512회 리매핑 발생
|
||||
4. 리매핑 8회 후 Field Diagnostics에서 문제 감지
|
||||
|
||||
[SRAM RMA 기준]
|
||||
1. SRAM Threshold Exceeded = Yes → 즉시 RMA
|
||||
2. Parity SRAM: 주소 뱅크 내 4개 이상 UCE Unique Count
|
||||
3. SECDED ECC SRAM: 주소 뱅크 내 2개 이상 UCE Unique Count
|
||||
"""
|
||||
verdict = RmaVerdict(
|
||||
gpu_index=gpu.index,
|
||||
gpu_name=gpu.name,
|
||||
serial_number=gpu.serial_number,
|
||||
)
|
||||
|
||||
# --- Row-Remapping 분석 ---
|
||||
total_remaps = gpu.remap_corr_error + gpu.remap_uncorr_error
|
||||
|
||||
verdict.row_remap_details = {
|
||||
"correctable_error": gpu.remap_corr_error,
|
||||
"uncorrectable_error": gpu.remap_uncorr_error,
|
||||
"total_remaps": total_remaps,
|
||||
"pending": gpu.remap_pending,
|
||||
"failure_occurred": gpu.remap_failure,
|
||||
"bank_max": gpu.remap_bank_max,
|
||||
"bank_high": gpu.remap_bank_high,
|
||||
"bank_partial": gpu.remap_bank_partial,
|
||||
"bank_low": gpu.remap_bank_low,
|
||||
"bank_none": gpu.remap_bank_none,
|
||||
}
|
||||
|
||||
# RMA 기준 1: Remapping Failure Occurred = Yes
|
||||
if gpu.remap_failure.lower() == "yes":
|
||||
verdict.qualifies_for_rma = True
|
||||
verdict.reasons.append(
|
||||
"Remapping Failure Occurred = Yes: "
|
||||
"리매핑 실패 플래그 설정됨. 즉시 RMA 대상."
|
||||
)
|
||||
|
||||
# RMA 기준 2: Uncorrectable Error Row 8회 이상 (Bank당)
|
||||
if gpu.remap_uncorr_error >= 8:
|
||||
verdict.qualifies_for_rma = True
|
||||
verdict.reasons.append(
|
||||
f"Uncorrectable Error 리매핑 {gpu.remap_uncorr_error}회: "
|
||||
f"Bank당 8회 이상 Uncorrectable Error 리매핑 발생 (기준: 8회)."
|
||||
)
|
||||
|
||||
# RMA 기준 3: 총 512회 리매핑
|
||||
if total_remaps >= 512:
|
||||
verdict.qualifies_for_rma = True
|
||||
verdict.reasons.append(
|
||||
f"총 리매핑 {total_remaps}회: "
|
||||
f"Uncorrectable Memory Error로 총 512회 이상 리매핑 발생 (기준: 512회)."
|
||||
)
|
||||
|
||||
# 경고: 리매핑 진행 중
|
||||
if gpu.remap_pending.lower() == "yes":
|
||||
verdict.warnings.append(
|
||||
"Row Remapping Pending = Yes: 리매핑 대기 중. 재부팅 후 적용됩니다."
|
||||
)
|
||||
|
||||
# 경고: Bank 가용성 낮음
|
||||
if gpu.remap_bank_none > 0:
|
||||
verdict.warnings.append(
|
||||
f"Bank Remap None = {gpu.remap_bank_none}: "
|
||||
f"리매핑 가용 뱅크가 소진된 뱅크가 있습니다."
|
||||
)
|
||||
if gpu.remap_bank_low > 0:
|
||||
verdict.warnings.append(
|
||||
f"Bank Remap Low = {gpu.remap_bank_low}: "
|
||||
f"리매핑 가용 뱅크가 적은 뱅크가 있습니다."
|
||||
)
|
||||
|
||||
# 경고: 리매핑이 상당수 발생
|
||||
if 0 < gpu.remap_uncorr_error < 8:
|
||||
verdict.warnings.append(
|
||||
f"Uncorrectable Error 리매핑 {gpu.remap_uncorr_error}회: "
|
||||
f"아직 RMA 기준(8회) 미달이나 모니터링 필요."
|
||||
)
|
||||
|
||||
# --- SRAM 분석 ---
|
||||
sram_uncorr_parity = _safe_int(gpu.sram_uncorr_parity_aggregate)
|
||||
sram_uncorr_secded = _safe_int(gpu.sram_uncorr_secded_aggregate)
|
||||
|
||||
verdict.sram_details = {
|
||||
"volatile": {
|
||||
"correctable": gpu.sram_corr_volatile,
|
||||
"uncorrectable_parity": gpu.sram_uncorr_parity_volatile,
|
||||
"uncorrectable_secded": gpu.sram_uncorr_secded_volatile,
|
||||
},
|
||||
"aggregate": {
|
||||
"correctable": gpu.sram_corr_aggregate,
|
||||
"uncorrectable_parity": gpu.sram_uncorr_parity_aggregate,
|
||||
"uncorrectable_secded": gpu.sram_uncorr_secded_aggregate,
|
||||
"threshold_exceeded": gpu.sram_threshold_exceeded,
|
||||
},
|
||||
"sources": {
|
||||
"l2": gpu.sram_l2,
|
||||
"sm": gpu.sram_sm,
|
||||
"microcontroller": gpu.sram_microcontroller,
|
||||
"pcie": gpu.sram_pcie,
|
||||
"other": gpu.sram_other,
|
||||
},
|
||||
"dram": {
|
||||
"corr_volatile": gpu.dram_corr_volatile,
|
||||
"uncorr_volatile": gpu.dram_uncorr_volatile,
|
||||
"corr_aggregate": gpu.dram_corr_aggregate,
|
||||
"uncorr_aggregate": gpu.dram_uncorr_aggregate,
|
||||
},
|
||||
}
|
||||
|
||||
# SRAM RMA 기준 1: Threshold Exceeded = Yes
|
||||
if gpu.sram_threshold_exceeded.lower() == "yes":
|
||||
verdict.qualifies_for_rma = True
|
||||
verdict.reasons.append(
|
||||
"SRAM Threshold Exceeded = Yes: "
|
||||
"SRAM UCE 임계치 초과 플래그 설정됨. 즉시 RMA 대상."
|
||||
)
|
||||
|
||||
# SRAM RMA 기준 2: Parity SRAM UCE >= 4
|
||||
if sram_uncorr_parity >= 4:
|
||||
verdict.qualifies_for_rma = True
|
||||
verdict.reasons.append(
|
||||
f"SRAM Uncorrectable Parity (Aggregate) = {sram_uncorr_parity}: "
|
||||
f"Parity 보호 SRAM에서 4개 이상 UCE 발생 (기준: 4개)."
|
||||
)
|
||||
|
||||
# SRAM RMA 기준 3: SECDED ECC SRAM UCE >= 2
|
||||
if sram_uncorr_secded >= 2:
|
||||
verdict.qualifies_for_rma = True
|
||||
verdict.reasons.append(
|
||||
f"SRAM Uncorrectable SEC-DED (Aggregate) = {sram_uncorr_secded}: "
|
||||
f"SECDED ECC 보호 SRAM에서 2개 이상 UCE 발생 (기준: 2개)."
|
||||
)
|
||||
|
||||
# 경고: SRAM UCE가 있지만 기준 미달
|
||||
if 0 < sram_uncorr_parity < 4:
|
||||
verdict.warnings.append(
|
||||
f"SRAM Parity UCE = {sram_uncorr_parity}: "
|
||||
f"아직 RMA 기준(4개) 미달이나 모니터링 필요."
|
||||
)
|
||||
if 0 < sram_uncorr_secded < 2:
|
||||
verdict.warnings.append(
|
||||
f"SRAM SECDED UCE = {sram_uncorr_secded}: "
|
||||
f"아직 RMA 기준(2개) 미달이나 모니터링 필요."
|
||||
)
|
||||
|
||||
return verdict
|
||||
@@ -0,0 +1,2 @@
|
||||
streamlit>=1.30.0
|
||||
pandas>=2.0.0
|
||||
Reference in New Issue
Block a user