Kim.KANGHEE/nvidia-bug
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: merge Xid-Catalog.xlsx, fix dmesg regex, and improve history UI

Browse Source
This commit is contained in:
unknown
2026-06-24 14:43:05 +09:00
commit 59e3a62ee5
13 changed files with 3913 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
log_parser.py
+997
View File
@@ -0,0 +1,997 @@
"""
NV-Log Analyzer - 핵심 분석 엔진
nvidia-bug-report.log.gz 파일을 파싱하여 XID 에러, GPU 상태, 커널 로그를 분석합니다.
성능 최적화: 모든 파싱을 라인 단위로 처리하여 대용량 파일에서도 빠르게 동작합니다.
"""
import gzip
import json
import re
from collections import Counter, defaultdict
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
# XID 데이터베이스 로드
DATA_DIR = Path(__file__).parent / "data"
with open(DATA_DIR / "xid_database.json", "r", encoding="utf-8") as f:
XID_DB = json.load(f)
# --- 데이터 클래스 ---
@dataclass
class XidEvent:
timestamp: str
xid_code: int
pci_address: str
raw_line: str
name: str = ""
severity: str = "info"
description: str = ""
action: str = ""
gpu_index: int = -1
gpu_name: str = ""
gpu_serial: str = ""
def __post_init__(self):
xid_info = XID_DB["xid_errors"].get(str(self.xid_code))
if xid_info:
self.name = xid_info["name"]
self.severity = xid_info["severity"]
self.description = xid_info["description"]
self.action = xid_info["action"]
else:
self.name = f"Unknown XID {self.xid_code}"
self.severity = "warning"
self.description = f"XID {self.xid_code}에 대한 정보가 데이터베이스에 없습니다."
self.action = "NVIDIA 공식 문서에서 해당 XID 코드를 확인하세요."
@dataclass
class GpuInfo:
index: int = 0
name: str = "N/A"
product_brand: str = "N/A"
product_architecture: str = "N/A"
driver_version: str = "N/A"
cuda_version: str = "N/A"
vbios_version: str = "N/A"
inforom_version: str = "N/A"
gpu_uuid: str = "N/A"
temperature: int = 0
temperature_limit: str = "N/A"
power_draw: float = 0.0
power_limit: float = 0.0
power_state: str = "N/A"
memory_used: int = 0
memory_total: int = 0
memory_free: int = 0
bar1_total: int = 0
bar1_used: int = 0
gpu_utilization: int = 0
memory_utilization: int = 0
ecc_mode: str = "N/A"
ecc_single_bit: int = 0
ecc_double_bit: int = 0
retired_pages_single: int = 0
retired_pages_double: int = 0
pci_address: str = "N/A"
pci_link_gen_current: str = "N/A"
pci_link_width_current: str = "N/A"
serial_number: str = "N/A"
fan_speed: int = 0
persistence_mode: str = "N/A"
compute_mode: str = "N/A"
mig_mode: str = "N/A"
clock_graphics: str = "N/A"
clock_memory: str = "N/A"
clock_max_graphics: str = "N/A"
clock_max_memory: str = "N/A"
# --- HW Slowdown ---
hw_slowdown: str = "N/A"
hw_thermal_slowdown: str = "N/A"
hw_power_brake_slowdown: str = "N/A"
# --- RMA 관련 필드 ---
# Row Remapping
remap_corr_error: int = 0
remap_uncorr_error: int = 0
remap_pending: str = "N/A"
remap_failure: str = "N/A"
remap_bank_max: int = 0
remap_bank_high: int = 0
remap_bank_partial: int = 0
remap_bank_low: int = 0
remap_bank_none: int = 0
# SRAM ECC
sram_corr_volatile: str = "N/A"
sram_uncorr_parity_volatile: str = "N/A"
sram_uncorr_secded_volatile: str = "N/A"
sram_corr_aggregate: str = "N/A"
sram_uncorr_parity_aggregate: str = "N/A"
sram_uncorr_secded_aggregate: str = "N/A"
sram_threshold_exceeded: str = "N/A"
# SRAM Sources
sram_l2: str = "N/A"
sram_sm: str = "N/A"
sram_microcontroller: str = "N/A"
sram_pcie: str = "N/A"
sram_other: str = "N/A"
# DRAM ECC
dram_corr_volatile: str = "N/A"
dram_uncorr_volatile: str = "N/A"
dram_corr_aggregate: str = "N/A"
dram_uncorr_aggregate: str = "N/A"
@dataclass
class RmaVerdict:
"""GPU별 RMA 판정 결과"""
gpu_index: int = 0
gpu_name: str = ""
serial_number: str = ""
qualifies_for_rma: bool = False
reasons: list = field(default_factory=list)
warnings: list = field(default_factory=list)
row_remap_details: dict = field(default_factory=dict)
sram_details: dict = field(default_factory=dict)
@dataclass
class KernelEvent:
timestamp: str
level: str
message: str
raw_line: str
@dataclass
class AnalysisResult:
filename: str = ""
file_size_mb: float = 0.0
analysis_time: str = ""
gpus: list = field(default_factory=list)
driver_version: str = "N/A"
cuda_version: str = "N/A"
xid_events: list = field(default_factory=list)
xid_summary: dict = field(default_factory=dict)
kernel_events: list = field(default_factory=list)
kernel_summary: dict = field(default_factory=dict)
ecc_info: dict = field(default_factory=dict)
rma_verdicts: list = field(default_factory=list) # List[RmaVerdict]
rma_count: int = 0 # RMA 대상 GPU 수
overall_status: str = "healthy"
total_issues: int = 0
critical_count: int = 0
warning_count: int = 0
info_count: int = 0
# --- 라인 단위 정규표현식 (성능 최적화) ---
# XID: 한 라인 안에서만 매칭
RE_XID = re.compile(r"NVRM:\s*Xid\s*\(PCI:([^)]+)\):\s*(\d+)")
RE_TIMESTAMP = re.compile(r"^(\w{3}\s+\d+\s+[\d:]+|\d{4}-\d{2}-\d{2}[T ][\d:]+|[\d]+\.[\d]+)")
# GPU 정보 (라인 단위) - nvidia-smi -q 출력의 "Key : Value" 패턴
RE_GPU_NAME = re.compile(r"Product Name\s*:\s*(.+)", re.IGNORECASE)
RE_GPU_BRAND = re.compile(r"Product Brand\s*:\s*(.+)", re.IGNORECASE)
RE_GPU_ARCH = re.compile(r"Product Architecture\s*:\s*(.+)", re.IGNORECASE)
RE_DRIVER = re.compile(r"Driver Version\s*:\s*([\d.]+)", re.IGNORECASE)
RE_CUDA = re.compile(r"CUDA Version\s*:\s*([\d.]+)", re.IGNORECASE)
RE_VBIOS = re.compile(r"VBIOS Version\s*:\s*(\S+)", re.IGNORECASE)
RE_INFOROM = re.compile(r"Image Version\s*:\s*(\S+)", re.IGNORECASE)
RE_UUID = re.compile(r"GPU UUID\s*:\s*(GPU-\S+)", re.IGNORECASE)
RE_TEMP = re.compile(r"GPU Current Temp\s*:\s*(\d+)\s*C", re.IGNORECASE)
RE_TEMP_LIMIT = re.compile(r"GPU T\.?Limit Temp\s*:\s*(.+)", re.IGNORECASE)
RE_POWER_DRAW = re.compile(r"Power Draw\s*:\s*([\d.]+)\s*W", re.IGNORECASE)
RE_POWER_LIMIT = re.compile(r"(?:Default |Enforced |Current )?Power Limit\s*:\s*([\d.]+)\s*W", re.IGNORECASE)
RE_POWER_STATE = re.compile(r"Power State\s*:\s*(\S+)", re.IGNORECASE)
RE_MEM_USED = re.compile(r"Used\s*:\s*(\d+)\s*MiB", re.IGNORECASE)
RE_MEM_TOTAL = re.compile(r"Total\s*:\s*(\d+)\s*MiB", re.IGNORECASE)
RE_MEM_FREE = re.compile(r"Free\s*:\s*(\d+)\s*MiB", re.IGNORECASE)
RE_GPU_UTIL = re.compile(r"Gpu\s*:\s*(\d+)\s*%", re.IGNORECASE)
RE_MEM_UTIL = re.compile(r"Memory\s*:\s*(\d+)\s*%", re.IGNORECASE)
RE_PCI = re.compile(r"Bus Id\s*:\s*([\dA-Fa-f:.]+)", re.IGNORECASE)
RE_PCI_GEN = re.compile(r"Current\s*:\s*Gen(\d+)", re.IGNORECASE)
RE_PCI_WIDTH = re.compile(r"Current\s*:\s*(\d+x)", re.IGNORECASE)
RE_SERIAL = re.compile(r"^\s*Serial Number\s*:\s*(\S+)", re.IGNORECASE)
RE_FAN = re.compile(r"Fan Speed\s*:\s*(\d+)\s*%", re.IGNORECASE)
RE_PERSIST = re.compile(r"Persistence Mode\s*:\s*(\w+)", re.IGNORECASE)
RE_COMPUTE_MODE = re.compile(r"Compute Mode\s*:\s*(.+)", re.IGNORECASE)
RE_MIG_MODE = re.compile(r"MIG Mode\s*:\s*(\w+)", re.IGNORECASE)
RE_ECC_MODE = re.compile(r"ECC Mode[\s\S]*?Current\s*:\s*(\w+)", re.IGNORECASE)
RE_CLOCK_GRAPHICS = re.compile(r"Graphics\s*:\s*(\d+)\s*MHz", re.IGNORECASE)
RE_CLOCK_MEMORY = re.compile(r"SM\s*:\s*(\d+)\s*MHz|Memory\s*:\s*(\d+)\s*MHz", re.IGNORECASE)
RE_HW_SLOWDOWN = re.compile(r"HW Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE)
RE_HW_THERMAL_SLOWDOWN = re.compile(r"HW Thermal Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE)
RE_HW_POWER_BRAKE_SLOWDOWN = re.compile(r"HW Power Brake Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE)
# GPU 섹션 헤더: "GPU 00000000:19:00.0" 또는 "GPU 0000:01:00.0" 또는 "GPU 0:"
RE_GPU_SECTION = re.compile(r"^GPU\s+([\dA-Fa-f]{4,8}:[\dA-Fa-f]{2}:[\dA-Fa-f]{2}\.\d)\s*$", re.IGNORECASE)
RE_GPU_SECTION_IDX = re.compile(r"^GPU\s+(\d+)\s*:\s*$", re.IGNORECASE)
# nvidia-smi 테이블 행
RE_SMI_ROW = re.compile(
r"\|\s*(\d+)\s+(.+?)\s+\w+\s*\|.*?(\d+)C.*?([\d.]+)W\s*/\s*([\d.]+)W.*?(\d+)MiB\s*/\s*(\d+)MiB.*?(\d+)%"
)
# ECC (라인 단위)
RE_SINGLE_BIT = re.compile(r"Single Bit", re.IGNORECASE)
RE_DOUBLE_BIT = re.compile(r"Double Bit", re.IGNORECASE)
RE_AGGREGATE = re.compile(r"Aggregate\s*:\s*(\d+)", re.IGNORECASE)
RE_RETIRED = re.compile(r"Retired pages\s*:\s*(\d+)", re.IGNORECASE)
RE_PENDING = re.compile(r"Pending\s*.*:\s*Yes", re.IGNORECASE)
# --- RMA 관련 정규표현식 ---
# Row Remapping
RE_REMAP_SECTION = re.compile(r"Remapped Rows", re.IGNORECASE)
RE_REMAP_CORR = re.compile(r"Correctable Error\s*:\s*(\d+)", re.IGNORECASE)
RE_REMAP_UNCORR = re.compile(r"Uncorrectable Error\s*:\s*(\d+)", re.IGNORECASE)
RE_REMAP_PENDING = re.compile(r"Pending\s*:\s*(Yes|No)", re.IGNORECASE)
RE_REMAP_FAILURE = re.compile(r"Remapping Failure Occurred\s*:\s*(Yes|No)", re.IGNORECASE)
RE_BANK_MAX = re.compile(r"Max\s*:\s*(\d+)\s*bank", re.IGNORECASE)
RE_BANK_HIGH = re.compile(r"High\s*:\s*(\d+)\s*bank", re.IGNORECASE)
RE_BANK_PARTIAL = re.compile(r"Partial\s*:\s*(\d+)\s*bank", re.IGNORECASE)
RE_BANK_LOW = re.compile(r"Low\s*:\s*(\d+)\s*bank", re.IGNORECASE)
RE_BANK_NONE = re.compile(r"None\s*:\s*(\d+)\s*bank", re.IGNORECASE)
# SRAM ECC
RE_SRAM_CORR = re.compile(r"SRAM Correctable\s*:\s*(\S+)", re.IGNORECASE)
RE_SRAM_UNCORR_PARITY = re.compile(r"SRAM Uncorrectable Parity\s*:\s*(\S+)", re.IGNORECASE)
RE_SRAM_UNCORR_SECDED = re.compile(r"SRAM Uncorrectable SEC-?DED\s*:\s*(\S+)", re.IGNORECASE)
RE_SRAM_THRESHOLD = re.compile(r"SRAM Threshold Exceeded\s*:\s*(\S+)", re.IGNORECASE)
# SRAM Sources
RE_SRAM_L2 = re.compile(r"SRAM L2\s*:\s*(\S+)", re.IGNORECASE)
RE_SRAM_SM = re.compile(r"SRAM SM\s*:\s*(\S+)", re.IGNORECASE)
RE_SRAM_MC = re.compile(r"SRAM Microcontroller\s*:\s*(\S+)", re.IGNORECASE)
RE_SRAM_PCIE = re.compile(r"SRAM PCIE\s*:\s*(\S+)", re.IGNORECASE)
RE_SRAM_OTHER = re.compile(r"SRAM Other\s*:\s*(\S+)", re.IGNORECASE)
# DRAM ECC
RE_DRAM_CORR = re.compile(r"DRAM Correctable\s*:\s*(\S+)", re.IGNORECASE)
RE_DRAM_UNCORR = re.compile(r"DRAM Uncorrectable\s*:\s*(\S+)", re.IGNORECASE)
# ECC Section context
RE_ECC_SECTION = re.compile(r"ECC Errors", re.IGNORECASE)
RE_VOLATILE = re.compile(r"^\s*Volatile\s*$", re.IGNORECASE)
RE_AGGREGATE_SECTION = re.compile(r"^\s*Aggregate\s*$", re.IGNORECASE)
RE_AGG_SRAM_SOURCES = re.compile(r"Aggregate Uncorrectable SRAM Sources", re.IGNORECASE)
# 커널 이벤트 (라인 단위 키워드 매칭)
KERNEL_CRITICAL_KW = re.compile(
r"(kernel panic|hardware error|machine check|fatal|BUG:|RIP:)", re.IGNORECASE
)
KERNEL_ERROR_KW = re.compile(
r"(error|failed|failure|segfault|cannot|unable to)", re.IGNORECASE
)
KERNEL_OOM_KW = re.compile(
r"(Out of memory|oom-killer|invoked oom|Killed process)", re.IGNORECASE
)
KERNEL_PCIE_KW = re.compile(
r"(AER|correctable error|uncorrectable error)", re.IGNORECASE
)
KERNEL_SKIP_KW = re.compile(r"(audit|systemd|session|polkit|dbus)", re.IGNORECASE)
# 섹션 구분: 언더스코어(_) 또는 대시(-)가 양옆에 있고 대괄호([, ])가 있을 수도 없을 수도 있는 구분선 매칭
# nvidia-bug-report.log 내 dmesg 등 다양한 로그 섹션 구분선을 감지합니다.
RE_SECTION = re.compile(r"^[_-]+\s*\[?(.+?)\]?\s*[_-]+$")
# --- 파서 함수들 ---
_RE_PCI_NORMALIZE = re.compile(
r"^([\dA-Fa-f]+):([\dA-Fa-f]{1,2}):([\dA-Fa-f]{1,2})(?:\.\d)?$"
)
_RE_PCI_NO_DOMAIN = re.compile(
r"^([\dA-Fa-f]{1,2}):([\dA-Fa-f]{1,2})(?:\.\d)?$"
)
def _norm_pci(addr: str) -> str:
"""
PCI 주소를 'DOMAIN:BUS:DEVICE' 형태로 정규화.
도메인은 정수값으로, bus/device는 소문자 2자리 hex로 통일.
- '0000:19:00' → '0:19:00'
- '00000000:19:00.0' → '0:19:00'
- '0001:3b:00.0' → '1:3b:00' (다른 도메인 → 다른 키)
- '19:00.0' → '0:19:00' (도메인 생략 → 0으로 가정)
매칭 실패 시 원문 소문자 반환.
"""
s = addr.strip()
m = _RE_PCI_NORMALIZE.match(s)
if m:
domain = int(m.group(1), 16)
bus = m.group(2).zfill(2).lower()
device = m.group(3).zfill(2).lower()
return f"{domain}:{bus}:{device}"
m2 = _RE_PCI_NO_DOMAIN.match(s)
if m2:
bus = m2.group(1).zfill(2).lower()
device = m2.group(2).zfill(2).lower()
return f"0:{bus}:{device}"
return s.lower()
def read_log_file(file_content: bytes, filename: str = "") -> str:
"""gz 또는 텍스트 파일을 읽어 문자열로 반환"""
if filename.endswith(".gz") or file_content[:2] == b'\x1f\x8b':
try:
return gzip.decompress(file_content).decode("utf-8", errors="replace")
except gzip.BadGzipFile:
return file_content.decode("utf-8", errors="replace")
return file_content.decode("utf-8", errors="replace")
def analyze_log(file_content: bytes, filename: str = "uploaded_file") -> AnalysisResult:
"""메인 분석 함수 - 라인 단위 싱글 패스 처리"""
result = AnalysisResult(
filename=filename,
file_size_mb=round(len(file_content) / (1024 * 1024), 2),
analysis_time=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
)
log_text = read_log_file(file_content, filename)
lines = log_text.splitlines()
# --- 상태 변수들 ---
current_section = "header"
driver_version = None
cuda_version = None
# GPU 파싱 상태
gpu_list = []
current_gpu = None
smi_gpus_found = False
# NVSMI LOG 상세 섹션의 PCI → Serial 별도 수집 (smi_gpus_found 여부 무관)
pci_serial_map: dict = {} # { "00000000:19:00.0" : "serial_string" }
_nvsmi_cur_pci: str = "" # 현재 GPU PCI (NVSMI 상세 파싱용)
# XID 이벤트
xid_events = []
# 커널 이벤트 (dmesg 섹션만)
kernel_events = []
seen_kernel = set()
in_dmesg = False
# RMA/ECC 파싱 상태 머신
in_ecc_section = False
ecc_subsection = None # "volatile", "aggregate", "agg_sources"
in_remap_section = False
in_bank_histogram = False
# ECC 상태
ecc_single_total = 0
ecc_double_total = 0
ecc_retired = 0
ecc_pending = False
ecc_context = None # "single" or "double" 마지막으로 본 ECC 유형
# --- 라인 단위 파싱 (싱글 패스) ---
for line in lines:
stripped = line.strip()
if not stripped:
continue
# 섹션 감지
sec_match = RE_SECTION.match(stripped)
if sec_match:
current_section = sec_match.group(1).strip().lower()
in_dmesg = any(kw in current_section for kw in ["dmesg", "kernel", "messages"])
continue
# --- NVSMI LOG: GPU PCI 헤더 및 Serial Number 항상 수집 ---
gpu_sec_always = RE_GPU_SECTION.match(stripped)
if gpu_sec_always:
_nvsmi_cur_pci = _norm_pci(gpu_sec_always.group(1))
elif _nvsmi_cur_pci:
serial_m = RE_SERIAL.search(stripped)
if serial_m and "chassis" not in stripped.lower():
val = serial_m.group(1).strip()
if val and val != "N/A":
pci_serial_map[_nvsmi_cur_pci] = val
# --- XID 에러 (모든 섹션) ---
xid_m = RE_XID.search(stripped)
if xid_m:
ts_m = RE_TIMESTAMP.match(stripped)
timestamp = ts_m.group(1) if ts_m else ""
xid_events.append(XidEvent(
timestamp=timestamp,
xid_code=int(xid_m.group(2)),
pci_address=xid_m.group(1).strip(),
raw_line=stripped[:500],
))
continue
# --- 드라이버/CUDA 버전 (최초 발견만) ---
if not driver_version:
dm = RE_DRIVER.search(stripped)
if dm:
driver_version = dm.group(1)
if not cuda_version:
cm = RE_CUDA.search(stripped)
if cm:
cuda_version = cm.group(1)
# --- nvidia-smi 테이블 행 ---
smi_m = RE_SMI_ROW.search(stripped)
if smi_m:
smi_gpus_found = True
gpu_list.append(GpuInfo(
index=int(smi_m.group(1)),
name=smi_m.group(2).strip(),
temperature=int(smi_m.group(3)),
power_draw=float(smi_m.group(4)),
power_limit=float(smi_m.group(5)),
memory_used=int(smi_m.group(6)),
memory_total=int(smi_m.group(7)),
gpu_utilization=int(smi_m.group(8)),
))
continue
# --- nvidia-smi -q 상세 형식 (GPU 섹션 기반) ---
if not smi_gpus_found:
# GPU 섹션 헤더 감지: "GPU 00000000:19:00.0" 또는 "GPU 0:"
gpu_sec_m = RE_GPU_SECTION.match(stripped) or RE_GPU_SECTION_IDX.match(stripped)
if gpu_sec_m:
if current_gpu and current_gpu.name != "N/A":
gpu_list.append(current_gpu)
current_gpu = GpuInfo(index=len(gpu_list))
# PCI 주소가 헤더에 포함된 경우
addr = gpu_sec_m.group(1)
if ":" in addr and "." in addr:
current_gpu.pci_address = addr
continue
if current_gpu is not None:
# 제품 정보
m = RE_GPU_NAME.search(stripped)
if m:
current_gpu.name = m.group(1).strip()
continue
m = RE_GPU_BRAND.search(stripped)
if m:
current_gpu.product_brand = m.group(1).strip()
continue
m = RE_GPU_ARCH.search(stripped)
if m:
current_gpu.product_architecture = m.group(1).strip()
continue
m = RE_UUID.search(stripped)
if m:
current_gpu.gpu_uuid = m.group(1).strip()
continue
m = RE_SERIAL.search(stripped)
if m and "chassis" not in stripped.lower():
val = m.group(1).strip()
if val != "N/A":
current_gpu.serial_number = val
continue
m = RE_VBIOS.search(stripped)
if m:
current_gpu.vbios_version = m.group(1).strip()
continue
m = RE_INFOROM.search(stripped)
if m:
current_gpu.inforom_version = m.group(1).strip()
continue
# 온도
m = RE_TEMP.search(stripped)
if m:
current_gpu.temperature = int(m.group(1))
continue
m = RE_TEMP_LIMIT.search(stripped)
if m:
current_gpu.temperature_limit = m.group(1).strip()
continue
# 전력
m = RE_POWER_DRAW.search(stripped)
if m:
current_gpu.power_draw = float(m.group(1))
continue
m = RE_POWER_LIMIT.search(stripped)
if m:
current_gpu.power_limit = float(m.group(1))
continue
m = RE_POWER_STATE.search(stripped)
if m:
current_gpu.power_state = m.group(1).strip()
continue
# 메모리 (FB Memory Usage 값만 유지, 이후 Conf Compute 등에 덮어쓰기 방지)
m = RE_MEM_TOTAL.search(stripped)
if m:
val = int(m.group(1))
if current_gpu.memory_total == 0 and val > 0:
current_gpu.memory_total = val
continue
m = RE_MEM_USED.search(stripped)
if m:
val = int(m.group(1))
if current_gpu.memory_used == 0:
current_gpu.memory_used = val
continue
m = RE_MEM_FREE.search(stripped)
if m:
val = int(m.group(1))
if current_gpu.memory_free == 0 and val > 0:
current_gpu.memory_free = val
continue
# 사용률
m = RE_GPU_UTIL.search(stripped)
if m:
current_gpu.gpu_utilization = int(m.group(1))
continue
m = RE_MEM_UTIL.search(stripped)
if m:
current_gpu.memory_utilization = int(m.group(1))
continue
# PCI
m = RE_PCI.search(stripped)
if m:
current_gpu.pci_address = m.group(1).strip()
continue
m = RE_PCI_GEN.search(stripped)
if m:
current_gpu.pci_link_gen_current = f"Gen{m.group(1)}"
continue
m = RE_PCI_WIDTH.search(stripped)
if m:
current_gpu.pci_link_width_current = m.group(1)
continue
# 기타
m = RE_FAN.search(stripped)
if m:
current_gpu.fan_speed = int(m.group(1))
continue
m = RE_PERSIST.search(stripped)
if m:
current_gpu.persistence_mode = m.group(1).strip()
continue
m = RE_COMPUTE_MODE.search(stripped)
if m:
current_gpu.compute_mode = m.group(1).strip()
continue
m = RE_MIG_MODE.search(stripped)
if m:
current_gpu.mig_mode = m.group(1).strip()
continue
m = RE_ECC_MODE.search(stripped)
if m:
current_gpu.ecc_mode = m.group(1).strip()
continue
m = RE_CLOCK_GRAPHICS.search(stripped)
if m:
current_gpu.clock_graphics = f"{m.group(1)} MHz"
continue
# HW Slowdown (순서 중요: 더 구체적인 패턴 먼저)
m = RE_HW_THERMAL_SLOWDOWN.search(stripped)
if m:
current_gpu.hw_thermal_slowdown = m.group(1)
continue
m = RE_HW_POWER_BRAKE_SLOWDOWN.search(stripped)
if m:
current_gpu.hw_power_brake_slowdown = m.group(1)
continue
m = RE_HW_SLOWDOWN.search(stripped)
if m:
current_gpu.hw_slowdown = m.group(1)
continue
# --- RMA: ECC 섹션 파싱 (GPU 내부) ---
if RE_ECC_SECTION.match(stripped):
in_ecc_section = True
ecc_subsection = None
continue
if in_ecc_section:
if RE_VOLATILE.match(stripped):
ecc_subsection = "volatile"
continue
if RE_AGGREGATE_SECTION.match(stripped):
ecc_subsection = "aggregate"
continue
if RE_AGG_SRAM_SOURCES.search(stripped):
ecc_subsection = "agg_sources"
continue
if ecc_subsection == "volatile":
m = RE_SRAM_CORR.search(stripped)
if m: current_gpu.sram_corr_volatile = m.group(1); continue
m = RE_SRAM_UNCORR_PARITY.search(stripped)
if m: current_gpu.sram_uncorr_parity_volatile = m.group(1); continue
m = RE_SRAM_UNCORR_SECDED.search(stripped)
if m: current_gpu.sram_uncorr_secded_volatile = m.group(1); continue
m = RE_DRAM_CORR.search(stripped)
if m: current_gpu.dram_corr_volatile = m.group(1); continue
m = RE_DRAM_UNCORR.search(stripped)
if m: current_gpu.dram_uncorr_volatile = m.group(1); continue
elif ecc_subsection == "aggregate":
m = RE_SRAM_CORR.search(stripped)
if m: current_gpu.sram_corr_aggregate = m.group(1); continue
m = RE_SRAM_UNCORR_PARITY.search(stripped)
if m: current_gpu.sram_uncorr_parity_aggregate = m.group(1); continue
m = RE_SRAM_UNCORR_SECDED.search(stripped)
if m: current_gpu.sram_uncorr_secded_aggregate = m.group(1); continue
m = RE_DRAM_CORR.search(stripped)
if m: current_gpu.dram_corr_aggregate = m.group(1); continue
m = RE_DRAM_UNCORR.search(stripped)
if m: current_gpu.dram_uncorr_aggregate = m.group(1); continue
m = RE_SRAM_THRESHOLD.search(stripped)
if m: current_gpu.sram_threshold_exceeded = m.group(1); continue
elif ecc_subsection == "agg_sources":
m = RE_SRAM_L2.search(stripped)
if m: current_gpu.sram_l2 = m.group(1); continue
m = RE_SRAM_SM.search(stripped)
if m: current_gpu.sram_sm = m.group(1); continue
m = RE_SRAM_MC.search(stripped)
if m: current_gpu.sram_microcontroller = m.group(1); continue
m = RE_SRAM_PCIE.search(stripped)
if m: current_gpu.sram_pcie = m.group(1); continue
m = RE_SRAM_OTHER.search(stripped)
if m: current_gpu.sram_other = m.group(1); continue
# --- RMA: Remapped Rows 섹션 파싱 (GPU 내부) ---
if RE_REMAP_SECTION.match(stripped):
in_remap_section = True
in_ecc_section = False
in_bank_histogram = False
continue
if in_remap_section:
m = RE_REMAP_CORR.search(stripped)
if m and "remap" not in stripped.lower():
current_gpu.remap_corr_error = int(m.group(1)); continue
m = RE_REMAP_UNCORR.search(stripped)
if m:
current_gpu.remap_uncorr_error = int(m.group(1)); continue
m = RE_REMAP_FAILURE.search(stripped)
if m:
current_gpu.remap_failure = m.group(1); continue
m = RE_REMAP_PENDING.search(stripped)
if m and "page" not in stripped.lower():
current_gpu.remap_pending = m.group(1); continue
if "Bank Remap Availability" in stripped:
in_bank_histogram = True; continue
if in_bank_histogram:
m = RE_BANK_MAX.search(stripped)
if m: current_gpu.remap_bank_max = int(m.group(1)); continue
m = RE_BANK_HIGH.search(stripped)
if m: current_gpu.remap_bank_high = int(m.group(1)); continue
m = RE_BANK_PARTIAL.search(stripped)
if m: current_gpu.remap_bank_partial = int(m.group(1)); continue
m = RE_BANK_LOW.search(stripped)
if m: current_gpu.remap_bank_low = int(m.group(1)); continue
m = RE_BANK_NONE.search(stripped)
if m:
current_gpu.remap_bank_none = int(m.group(1))
in_remap_section = False
in_bank_histogram = False
continue
# --- ECC 에러 전체 집계 (라인 단위 상태 머신) ---
if RE_SINGLE_BIT.search(stripped) and "sram" not in stripped.lower():
ecc_context = "single"
continue
if RE_DOUBLE_BIT.search(stripped) and "sram" not in stripped.lower():
ecc_context = "double"
continue
if ecc_context:
agg_m = RE_AGGREGATE.search(stripped)
if agg_m:
val = int(agg_m.group(1))
if ecc_context == "single":
ecc_single_total = max(ecc_single_total, val)
else:
ecc_double_total = max(ecc_double_total, val)
ecc_context = None
continue
ret_m = RE_RETIRED.search(stripped)
if ret_m:
ecc_retired = max(ecc_retired, int(ret_m.group(1)))
continue
if RE_PENDING.search(stripped) and "remap" not in stripped.lower():
ecc_pending = True
continue
# --- 커널 이벤트 (dmesg 섹션) ---
if in_dmesg and stripped not in seen_kernel:
if KERNEL_SKIP_KW.search(stripped):
continue
level = None
keyword = ""
# 우선순위: oom > critical > pcie > error
m = KERNEL_OOM_KW.search(stripped)
if m:
level, keyword = "oom", m.group(1)
if not level:
m = KERNEL_CRITICAL_KW.search(stripped)
if m:
level, keyword = "critical", m.group(1)
if not level:
m = KERNEL_PCIE_KW.search(stripped)
if m:
level, keyword = "pcie_error", m.group(1)
if not level:
m = KERNEL_ERROR_KW.search(stripped)
if m:
level, keyword = "error", m.group(1)
if level:
seen_kernel.add(stripped)
ts_m = RE_TIMESTAMP.match(stripped)
kernel_events.append(KernelEvent(
timestamp=ts_m.group(1) if ts_m else "",
level=level,
message=keyword,
raw_line=stripped[:500],
))
# --- 마지막 GPU 저장 ---
if current_gpu and current_gpu.name != "N/A" and not smi_gpus_found:
gpu_list.append(current_gpu)
# --- 결과 조립 ---
result.driver_version = driver_version or "N/A"
result.cuda_version = cuda_version or "N/A"
for gpu in gpu_list:
gpu.driver_version = result.driver_version
gpu.cuda_version = result.cuda_version
result.gpus = gpu_list
# gpu_list 시리얼 보완: NVSMI LOG에서 수집한 pci_serial_map 활용
for gpu in gpu_list:
if (not gpu.serial_number or gpu.serial_number == "N/A") and gpu.pci_address:
s = pci_serial_map.get(_norm_pci(gpu.pci_address))
if s:
gpu.serial_number = s
# PCI 주소 → GPU 매핑 (정규화 키 사용)
pci_to_gpu = {}
for gpu in gpu_list:
if gpu.pci_address and gpu.pci_address != "N/A":
pci_to_gpu[_norm_pci(gpu.pci_address)] = gpu
for event in xid_events:
key = _norm_pci(event.pci_address)
matched = pci_to_gpu.get(key)
if matched:
event.gpu_index = matched.index
event.gpu_name = matched.name
event.gpu_serial = matched.serial_number or pci_serial_map.get(key, "")
else:
# gpu_list에 없어도 pci_serial_map에서 시리얼 보완
event.gpu_serial = pci_serial_map.get(key, "")
result.xid_events = xid_events
xid_counter = Counter(e.xid_code for e in xid_events)
result.xid_summary = {
code: {
"count": count,
"severity": XID_DB["xid_errors"].get(str(code), {}).get("severity", "warning"),
"name": XID_DB["xid_errors"].get(str(code), {}).get("name", f"Unknown XID {code}"),
"gpus": sorted({
f"GPU {e.gpu_index} {e.gpu_name} [S/N: {e.gpu_serial}]" if e.gpu_serial
else (f"GPU {e.gpu_index} ({e.gpu_name})" if e.gpu_name else e.pci_address)
for e in xid_events if e.xid_code == code
}),
}
for code, count in xid_counter.most_common()
}
result.kernel_events = kernel_events
kernel_counter = Counter(e.level for e in kernel_events)
result.kernel_summary = dict(kernel_counter)
result.ecc_info = {
"single_bit_total": ecc_single_total,
"double_bit_total": ecc_double_total,
"retired_pages": ecc_retired,
"pending_retirement": ecc_pending,
}
# --- RMA 판정 ---
result.rma_verdicts = [evaluate_rma(gpu) for gpu in gpu_list]
result.rma_count = sum(1 for v in result.rma_verdicts if v.qualifies_for_rma)
# 상태 판정
critical_xids = sum(1 for e in xid_events if e.severity == "critical")
warning_xids = sum(1 for e in xid_events if e.severity == "warning")
result.critical_count = (
critical_xids
+ kernel_counter.get("critical", 0)
+ kernel_counter.get("oom", 0)
+ (1 if ecc_double_total > 0 else 0)
+ result.rma_count # RMA 대상 GPU 수도 critical에 포함
)
result.warning_count = (
warning_xids
+ kernel_counter.get("error", 0)
+ kernel_counter.get("pcie_error", 0)
+ (1 if ecc_single_total > 10 else 0)
)
result.info_count = sum(1 for e in xid_events if e.severity == "info")
result.total_issues = result.critical_count + result.warning_count
if result.rma_count > 0:
result.overall_status = "critical"
elif result.critical_count > 0:
result.overall_status = "critical"
elif result.warning_count > 0:
result.overall_status = "warning"
else:
result.overall_status = "healthy"
return result
def _safe_int(val: str) -> int:
"""N/A 등 숫자가 아닌 값을 0으로 변환"""
try:
return int(val)
except (ValueError, TypeError):
return 0
def evaluate_rma(gpu: GpuInfo) -> RmaVerdict:
"""
NVIDIA Hopper RMA 기준에 따라 GPU의 RMA 적격 여부를 판정합니다.
[Row-Remapping RMA 기준]
1. Remapping Failure Occurred = Yes → 즉시 RMA
2. 한 Bank에서 Uncorrectable Error Row 8회 이상 리매핑
3. 총 512회 리매핑 발생
4. 리매핑 8회 후 Field Diagnostics에서 문제 감지
[SRAM RMA 기준]
1. SRAM Threshold Exceeded = Yes → 즉시 RMA
2. Parity SRAM: 주소 뱅크 내 4개 이상 UCE Unique Count
3. SECDED ECC SRAM: 주소 뱅크 내 2개 이상 UCE Unique Count
"""
verdict = RmaVerdict(
gpu_index=gpu.index,
gpu_name=gpu.name,
serial_number=gpu.serial_number,
)
# --- Row-Remapping 분석 ---
total_remaps = gpu.remap_corr_error + gpu.remap_uncorr_error
verdict.row_remap_details = {
"correctable_error": gpu.remap_corr_error,
"uncorrectable_error": gpu.remap_uncorr_error,
"total_remaps": total_remaps,
"pending": gpu.remap_pending,
"failure_occurred": gpu.remap_failure,
"bank_max": gpu.remap_bank_max,
"bank_high": gpu.remap_bank_high,
"bank_partial": gpu.remap_bank_partial,
"bank_low": gpu.remap_bank_low,
"bank_none": gpu.remap_bank_none,
}
# RMA 기준 1: Remapping Failure Occurred = Yes
if gpu.remap_failure.lower() == "yes":
verdict.qualifies_for_rma = True
verdict.reasons.append(
"Remapping Failure Occurred = Yes: "
"리매핑 실패 플래그 설정됨. 즉시 RMA 대상."
)
# RMA 기준 2: Uncorrectable Error Row 8회 이상 (Bank당)
if gpu.remap_uncorr_error >= 8:
verdict.qualifies_for_rma = True
verdict.reasons.append(
f"Uncorrectable Error 리매핑 {gpu.remap_uncorr_error}회: "
f"Bank당 8회 이상 Uncorrectable Error 리매핑 발생 (기준: 8회)."
)
# RMA 기준 3: 총 512회 리매핑
if total_remaps >= 512:
verdict.qualifies_for_rma = True
verdict.reasons.append(
f"총 리매핑 {total_remaps}회: "
f"Uncorrectable Memory Error로 총 512회 이상 리매핑 발생 (기준: 512회)."
)
# 경고: 리매핑 진행 중
if gpu.remap_pending.lower() == "yes":
verdict.warnings.append(
"Row Remapping Pending = Yes: 리매핑 대기 중. 재부팅 후 적용됩니다."
)
# 경고: Bank 가용성 낮음
if gpu.remap_bank_none > 0:
verdict.warnings.append(
f"Bank Remap None = {gpu.remap_bank_none}: "
f"리매핑 가용 뱅크가 소진된 뱅크가 있습니다."
)
if gpu.remap_bank_low > 0:
verdict.warnings.append(
f"Bank Remap Low = {gpu.remap_bank_low}: "
f"리매핑 가용 뱅크가 적은 뱅크가 있습니다."
)
# 경고: 리매핑이 상당수 발생
if 0 < gpu.remap_uncorr_error < 8:
verdict.warnings.append(
f"Uncorrectable Error 리매핑 {gpu.remap_uncorr_error}회: "
f"아직 RMA 기준(8회) 미달이나 모니터링 필요."
)
# --- SRAM 분석 ---
sram_uncorr_parity = _safe_int(gpu.sram_uncorr_parity_aggregate)
sram_uncorr_secded = _safe_int(gpu.sram_uncorr_secded_aggregate)
verdict.sram_details = {
"volatile": {
"correctable": gpu.sram_corr_volatile,
"uncorrectable_parity": gpu.sram_uncorr_parity_volatile,
"uncorrectable_secded": gpu.sram_uncorr_secded_volatile,
},
"aggregate": {
"correctable": gpu.sram_corr_aggregate,
"uncorrectable_parity": gpu.sram_uncorr_parity_aggregate,
"uncorrectable_secded": gpu.sram_uncorr_secded_aggregate,
"threshold_exceeded": gpu.sram_threshold_exceeded,
},
"sources": {
"l2": gpu.sram_l2,
"sm": gpu.sram_sm,
"microcontroller": gpu.sram_microcontroller,
"pcie": gpu.sram_pcie,
"other": gpu.sram_other,
},
"dram": {
"corr_volatile": gpu.dram_corr_volatile,
"uncorr_volatile": gpu.dram_uncorr_volatile,
"corr_aggregate": gpu.dram_corr_aggregate,
"uncorr_aggregate": gpu.dram_uncorr_aggregate,
},
}
# SRAM RMA 기준 1: Threshold Exceeded = Yes
if gpu.sram_threshold_exceeded.lower() == "yes":
verdict.qualifies_for_rma = True
verdict.reasons.append(
"SRAM Threshold Exceeded = Yes: "
"SRAM UCE 임계치 초과 플래그 설정됨. 즉시 RMA 대상."
)
# SRAM RMA 기준 2: Parity SRAM UCE >= 4
if sram_uncorr_parity >= 4:
verdict.qualifies_for_rma = True
verdict.reasons.append(
f"SRAM Uncorrectable Parity (Aggregate) = {sram_uncorr_parity}: "
f"Parity 보호 SRAM에서 4개 이상 UCE 발생 (기준: 4개)."
)
# SRAM RMA 기준 3: SECDED ECC SRAM UCE >= 2
if sram_uncorr_secded >= 2:
verdict.qualifies_for_rma = True
verdict.reasons.append(
f"SRAM Uncorrectable SEC-DED (Aggregate) = {sram_uncorr_secded}: "
f"SECDED ECC 보호 SRAM에서 2개 이상 UCE 발생 (기준: 2개)."
)
# 경고: SRAM UCE가 있지만 기준 미달
if 0 < sram_uncorr_parity < 4:
verdict.warnings.append(
f"SRAM Parity UCE = {sram_uncorr_parity}: "
f"아직 RMA 기준(4개) 미달이나 모니터링 필요."
)
if 0 < sram_uncorr_secded < 2:
verdict.warnings.append(
f"SRAM SECDED UCE = {sram_uncorr_secded}: "
f"아직 RMA 기준(2개) 미달이나 모니터링 필요."
)
return verdict