""" NV-Log Analyzer - 핵심 분석 엔진 nvidia-bug-report.log.gz 파일을 파싱하여 XID 에러, GPU 상태, 커널 로그를 분석합니다. 성능 최적화: 모든 파싱을 라인 단위로 처리하여 대용량 파일에서도 빠르게 동작합니다. """ import gzip import json import re from collections import Counter, defaultdict from dataclasses import dataclass, field from datetime import datetime from pathlib import Path # XID 데이터베이스 로드 DATA_DIR = Path(__file__).parent / "data" with open(DATA_DIR / "xid_database.json", "r", encoding="utf-8") as f: XID_DB = json.load(f) # --- 데이터 클래스 --- @dataclass class XidEvent: timestamp: str xid_code: int pci_address: str raw_line: str name: str = "" severity: str = "info" description: str = "" action: str = "" action_ko: str = "" action_en: str = "" gpu_index: int = -1 gpu_name: str = "" gpu_serial: str = "" def __post_init__(self): xid_info = XID_DB["xid_errors"].get(str(self.xid_code)) if xid_info: self.name = xid_info["name"] self.severity = xid_info["severity"] self.description = xid_info["description"] self.action = xid_info.get("action_ko", xid_info.get("action", "")) self.action_ko = xid_info.get("action_ko", xid_info.get("action", "")) self.action_en = xid_info.get("action_en", "NVIDIA official documentation does not specify a detailed action for this code.") else: self.name = f"Unknown XID {self.xid_code}" self.severity = "warning" self.description = f"XID {self.xid_code}에 대한 정보가 데이터베이스에 없습니다." self.action = "NVIDIA 공식 문서에서 해당 XID 코드를 확인하세요." self.action_ko = "NVIDIA 공식 문서에서 해당 XID 코드를 확인하세요." self.action_en = "Please check this XID code in NVIDIA official documentation." @dataclass class GpuInfo: index: int = 0 name: str = "N/A" product_brand: str = "N/A" product_architecture: str = "N/A" driver_version: str = "N/A" cuda_version: str = "N/A" vbios_version: str = "N/A" inforom_version: str = "N/A" gpu_uuid: str = "N/A" temperature: int = 0 temperature_limit: str = "N/A" power_draw: float = 0.0 power_limit: float = 0.0 power_state: str = "N/A" memory_used: int = 0 memory_total: int = 0 memory_free: int = 0 bar1_total: int = 0 bar1_used: int = 0 gpu_utilization: int = 0 memory_utilization: int = 0 ecc_mode: str = "N/A" ecc_single_bit: int = 0 ecc_double_bit: int = 0 retired_pages_single: int = 0 retired_pages_double: int = 0 pci_address: str = "N/A" pci_link_gen_current: str = "N/A" pci_link_width_current: str = "N/A" serial_number: str = "N/A" fan_speed: int = 0 persistence_mode: str = "N/A" compute_mode: str = "N/A" mig_mode: str = "N/A" clock_graphics: str = "N/A" clock_memory: str = "N/A" clock_max_graphics: str = "N/A" clock_max_memory: str = "N/A" # --- HW Slowdown --- hw_slowdown: str = "N/A" hw_thermal_slowdown: str = "N/A" hw_power_brake_slowdown: str = "N/A" # --- RMA 관련 필드 --- # Row Remapping remap_corr_error: int = 0 remap_uncorr_error: int = 0 remap_pending: str = "N/A" remap_failure: str = "N/A" remap_bank_max: int = 0 remap_bank_high: int = 0 remap_bank_partial: int = 0 remap_bank_low: int = 0 remap_bank_none: int = 0 # SRAM ECC sram_corr_volatile: str = "N/A" sram_uncorr_parity_volatile: str = "N/A" sram_uncorr_secded_volatile: str = "N/A" sram_corr_aggregate: str = "N/A" sram_uncorr_parity_aggregate: str = "N/A" sram_uncorr_secded_aggregate: str = "N/A" sram_threshold_exceeded: str = "N/A" # SRAM Sources sram_l2: str = "N/A" sram_sm: str = "N/A" sram_microcontroller: str = "N/A" sram_pcie: str = "N/A" sram_other: str = "N/A" # DRAM ECC dram_corr_volatile: str = "N/A" dram_uncorr_volatile: str = "N/A" dram_corr_aggregate: str = "N/A" dram_uncorr_aggregate: str = "N/A" @dataclass class RmaVerdict: """GPU별 RMA 판정 결과""" gpu_index: int = 0 gpu_name: str = "" serial_number: str = "" qualifies_for_rma: bool = False reasons: list = field(default_factory=list) warnings: list = field(default_factory=list) row_remap_details: dict = field(default_factory=dict) sram_details: dict = field(default_factory=dict) @dataclass class KernelEvent: timestamp: str level: str message: str raw_line: str @dataclass class AnalysisResult: filename: str = "" file_size_mb: float = 0.0 analysis_time: str = "" gpus: list = field(default_factory=list) driver_version: str = "N/A" cuda_version: str = "N/A" xid_events: list = field(default_factory=list) xid_summary: dict = field(default_factory=dict) kernel_events: list = field(default_factory=list) kernel_summary: dict = field(default_factory=dict) ecc_info: dict = field(default_factory=dict) rma_verdicts: list = field(default_factory=list) # List[RmaVerdict] rma_count: int = 0 # RMA 대상 GPU 수 overall_status: str = "healthy" total_issues: int = 0 critical_count: int = 0 warning_count: int = 0 info_count: int = 0 # --- 라인 단위 정규표현식 (성능 최적화) --- # XID: 한 라인 안에서만 매칭 RE_XID = re.compile(r"NVRM:\s*Xid\s*\(PCI:([^)]+)\):\s*(\d+)") RE_TIMESTAMP = re.compile(r"^(\w{3}\s+\d+\s+[\d:]+|\d{4}-\d{2}-\d{2}[T ][\d:]+|[\d]+\.[\d]+)") # GPU 정보 (라인 단위) - nvidia-smi -q 출력의 "Key : Value" 패턴 RE_GPU_NAME = re.compile(r"Product Name\s*:\s*(.+)", re.IGNORECASE) RE_GPU_BRAND = re.compile(r"Product Brand\s*:\s*(.+)", re.IGNORECASE) RE_GPU_ARCH = re.compile(r"Product Architecture\s*:\s*(.+)", re.IGNORECASE) RE_DRIVER = re.compile(r"Driver Version\s*:\s*([\d.]+)", re.IGNORECASE) RE_CUDA = re.compile(r"CUDA Version\s*:\s*([\d.]+)", re.IGNORECASE) RE_VBIOS = re.compile(r"VBIOS Version\s*:\s*(\S+)", re.IGNORECASE) RE_INFOROM = re.compile(r"Image Version\s*:\s*(\S+)", re.IGNORECASE) RE_UUID = re.compile(r"GPU UUID\s*:\s*(GPU-\S+)", re.IGNORECASE) RE_TEMP = re.compile(r"GPU Current Temp\s*:\s*(\d+)\s*C", re.IGNORECASE) RE_TEMP_LIMIT = re.compile(r"GPU T\.?Limit Temp\s*:\s*(.+)", re.IGNORECASE) RE_POWER_DRAW = re.compile(r"Power Draw\s*:\s*([\d.]+)\s*W", re.IGNORECASE) RE_POWER_LIMIT = re.compile(r"(?:Default |Enforced |Current )?Power Limit\s*:\s*([\d.]+)\s*W", re.IGNORECASE) RE_POWER_STATE = re.compile(r"Power State\s*:\s*(\S+)", re.IGNORECASE) RE_MEM_USED = re.compile(r"Used\s*:\s*(\d+)\s*MiB", re.IGNORECASE) RE_MEM_TOTAL = re.compile(r"Total\s*:\s*(\d+)\s*MiB", re.IGNORECASE) RE_MEM_FREE = re.compile(r"Free\s*:\s*(\d+)\s*MiB", re.IGNORECASE) RE_GPU_UTIL = re.compile(r"Gpu\s*:\s*(\d+)\s*%", re.IGNORECASE) RE_MEM_UTIL = re.compile(r"Memory\s*:\s*(\d+)\s*%", re.IGNORECASE) RE_PCI = re.compile(r"Bus Id\s*:\s*([\dA-Fa-f:.]+)", re.IGNORECASE) RE_PCI_GEN = re.compile(r"Current\s*:\s*Gen(\d+)", re.IGNORECASE) RE_PCI_WIDTH = re.compile(r"Current\s*:\s*(\d+x)", re.IGNORECASE) RE_SERIAL = re.compile(r"^\s*Serial Number\s*:\s*(\S+)", re.IGNORECASE) RE_FAN = re.compile(r"Fan Speed\s*:\s*(\d+)\s*%", re.IGNORECASE) RE_PERSIST = re.compile(r"Persistence Mode\s*:\s*(\w+)", re.IGNORECASE) RE_COMPUTE_MODE = re.compile(r"Compute Mode\s*:\s*(.+)", re.IGNORECASE) RE_MIG_MODE = re.compile(r"MIG Mode\s*:\s*(\w+)", re.IGNORECASE) RE_ECC_MODE = re.compile(r"ECC Mode[\s\S]*?Current\s*:\s*(\w+)", re.IGNORECASE) RE_CLOCK_GRAPHICS = re.compile(r"Graphics\s*:\s*(\d+)\s*MHz", re.IGNORECASE) RE_CLOCK_MEMORY = re.compile(r"SM\s*:\s*(\d+)\s*MHz|Memory\s*:\s*(\d+)\s*MHz", re.IGNORECASE) RE_HW_SLOWDOWN = re.compile(r"HW Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE) RE_HW_THERMAL_SLOWDOWN = re.compile(r"HW Thermal Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE) RE_HW_POWER_BRAKE_SLOWDOWN = re.compile(r"HW Power Brake Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE) # GPU 섹션 헤더: "GPU 00000000:19:00.0" 또는 "GPU 0000:01:00.0" 또는 "GPU 0:" RE_GPU_SECTION = re.compile(r"^GPU\s+([\dA-Fa-f]{4,8}:[\dA-Fa-f]{2}:[\dA-Fa-f]{2}\.\d)\s*$", re.IGNORECASE) RE_GPU_SECTION_IDX = re.compile(r"^GPU\s+(\d+)\s*:\s*$", re.IGNORECASE) # nvidia-smi 테이블 행 RE_SMI_ROW = re.compile( r"\|\s*(\d+)\s+(.+?)\s+\w+\s*\|.*?(\d+)C.*?([\d.]+)W\s*/\s*([\d.]+)W.*?(\d+)MiB\s*/\s*(\d+)MiB.*?(\d+)%" ) # ECC (라인 단위) RE_SINGLE_BIT = re.compile(r"Single Bit", re.IGNORECASE) RE_DOUBLE_BIT = re.compile(r"Double Bit", re.IGNORECASE) RE_AGGREGATE = re.compile(r"Aggregate\s*:\s*(\d+)", re.IGNORECASE) RE_RETIRED = re.compile(r"Retired pages\s*:\s*(\d+)", re.IGNORECASE) RE_PENDING = re.compile(r"Pending\s*.*:\s*Yes", re.IGNORECASE) # --- RMA 관련 정규표현식 --- # Row Remapping RE_REMAP_SECTION = re.compile(r"Remapped Rows", re.IGNORECASE) RE_REMAP_CORR = re.compile(r"Correctable Error\s*:\s*(\d+)", re.IGNORECASE) RE_REMAP_UNCORR = re.compile(r"Uncorrectable Error\s*:\s*(\d+)", re.IGNORECASE) RE_REMAP_PENDING = re.compile(r"Pending\s*:\s*(Yes|No)", re.IGNORECASE) RE_REMAP_FAILURE = re.compile(r"Remapping Failure Occurred\s*:\s*(Yes|No)", re.IGNORECASE) RE_BANK_MAX = re.compile(r"Max\s*:\s*(\d+)\s*bank", re.IGNORECASE) RE_BANK_HIGH = re.compile(r"High\s*:\s*(\d+)\s*bank", re.IGNORECASE) RE_BANK_PARTIAL = re.compile(r"Partial\s*:\s*(\d+)\s*bank", re.IGNORECASE) RE_BANK_LOW = re.compile(r"Low\s*:\s*(\d+)\s*bank", re.IGNORECASE) RE_BANK_NONE = re.compile(r"None\s*:\s*(\d+)\s*bank", re.IGNORECASE) # SRAM ECC RE_SRAM_CORR = re.compile(r"SRAM Correctable\s*:\s*(\S+)", re.IGNORECASE) RE_SRAM_UNCORR_PARITY = re.compile(r"SRAM Uncorrectable Parity\s*:\s*(\S+)", re.IGNORECASE) RE_SRAM_UNCORR_SECDED = re.compile(r"SRAM Uncorrectable SEC-?DED\s*:\s*(\S+)", re.IGNORECASE) RE_SRAM_THRESHOLD = re.compile(r"SRAM Threshold Exceeded\s*:\s*(\S+)", re.IGNORECASE) # SRAM Sources RE_SRAM_L2 = re.compile(r"SRAM L2\s*:\s*(\S+)", re.IGNORECASE) RE_SRAM_SM = re.compile(r"SRAM SM\s*:\s*(\S+)", re.IGNORECASE) RE_SRAM_MC = re.compile(r"SRAM Microcontroller\s*:\s*(\S+)", re.IGNORECASE) RE_SRAM_PCIE = re.compile(r"SRAM PCIE\s*:\s*(\S+)", re.IGNORECASE) RE_SRAM_OTHER = re.compile(r"SRAM Other\s*:\s*(\S+)", re.IGNORECASE) # DRAM ECC RE_DRAM_CORR = re.compile(r"DRAM Correctable\s*:\s*(\S+)", re.IGNORECASE) RE_DRAM_UNCORR = re.compile(r"DRAM Uncorrectable\s*:\s*(\S+)", re.IGNORECASE) # ECC Section context RE_ECC_SECTION = re.compile(r"ECC Errors", re.IGNORECASE) RE_VOLATILE = re.compile(r"^\s*Volatile\s*$", re.IGNORECASE) RE_AGGREGATE_SECTION = re.compile(r"^\s*Aggregate\s*$", re.IGNORECASE) RE_AGG_SRAM_SOURCES = re.compile(r"Aggregate Uncorrectable SRAM Sources", re.IGNORECASE) # 커널 이벤트 (라인 단위 키워드 매칭) KERNEL_CRITICAL_KW = re.compile( r"(kernel panic|hardware error|machine check|fatal|BUG:|RIP:)", re.IGNORECASE ) KERNEL_ERROR_KW = re.compile( r"(error|failed|failure|segfault|cannot|unable to)", re.IGNORECASE ) KERNEL_OOM_KW = re.compile( r"(Out of memory|oom-killer|invoked oom|Killed process)", re.IGNORECASE ) KERNEL_PCIE_KW = re.compile( r"(AER|correctable error|uncorrectable error)", re.IGNORECASE ) KERNEL_SKIP_KW = re.compile(r"(audit|systemd|session|polkit|dbus)", re.IGNORECASE) # 섹션 구분: 언더스코어(_) 또는 대시(-)가 양옆에 있고 대괄호([, ])가 있을 수도 없을 수도 있는 구분선 매칭 # nvidia-bug-report.log 내 dmesg 등 다양한 로그 섹션 구분선을 감지합니다. RE_SECTION = re.compile(r"^[_-]+\s*\[?(.+?)\]?\s*[_-]+$") # --- 파서 함수들 --- _RE_PCI_NORMALIZE = re.compile( r"^([\dA-Fa-f]+):([\dA-Fa-f]{1,2}):([\dA-Fa-f]{1,2})(?:\.\d)?$" ) _RE_PCI_NO_DOMAIN = re.compile( r"^([\dA-Fa-f]{1,2}):([\dA-Fa-f]{1,2})(?:\.\d)?$" ) def _norm_pci(addr: str) -> str: """ PCI 주소를 'DOMAIN:BUS:DEVICE' 형태로 정규화. 도메인은 정수값으로, bus/device는 소문자 2자리 hex로 통일. - '0000:19:00' → '0:19:00' - '00000000:19:00.0' → '0:19:00' - '0001:3b:00.0' → '1:3b:00' (다른 도메인 → 다른 키) - '19:00.0' → '0:19:00' (도메인 생략 → 0으로 가정) 매칭 실패 시 원문 소문자 반환. """ s = addr.strip() m = _RE_PCI_NORMALIZE.match(s) if m: domain = int(m.group(1), 16) bus = m.group(2).zfill(2).lower() device = m.group(3).zfill(2).lower() return f"{domain}:{bus}:{device}" m2 = _RE_PCI_NO_DOMAIN.match(s) if m2: bus = m2.group(1).zfill(2).lower() device = m2.group(2).zfill(2).lower() return f"0:{bus}:{device}" return s.lower() def read_log_file(file_content: bytes, filename: str = "") -> str: """gz 또는 텍스트 파일을 읽어 문자열로 반환""" if filename.endswith(".gz") or file_content[:2] == b'\x1f\x8b': try: return gzip.decompress(file_content).decode("utf-8", errors="replace") except gzip.BadGzipFile: return file_content.decode("utf-8", errors="replace") return file_content.decode("utf-8", errors="replace") def analyze_log(file_content: bytes, filename: str = "uploaded_file") -> AnalysisResult: """메인 분석 함수 - 라인 단위 싱글 패스 처리""" result = AnalysisResult( filename=filename, file_size_mb=round(len(file_content) / (1024 * 1024), 2), analysis_time=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), ) log_text = read_log_file(file_content, filename) lines = log_text.splitlines() # --- 상태 변수들 --- current_section = "header" driver_version = None cuda_version = None # GPU 파싱 상태 gpu_list = [] current_gpu = None smi_gpus_found = False # NVSMI LOG 상세 섹션의 PCI → Serial 별도 수집 (smi_gpus_found 여부 무관) pci_serial_map: dict = {} # { "00000000:19:00.0" : "serial_string" } _nvsmi_cur_pci: str = "" # 현재 GPU PCI (NVSMI 상세 파싱용) # XID 이벤트 xid_events = [] # 커널 이벤트 (dmesg 섹션만) kernel_events = [] seen_kernel = set() in_dmesg = False # RMA/ECC 파싱 상태 머신 in_ecc_section = False ecc_subsection = None # "volatile", "aggregate", "agg_sources" in_remap_section = False in_bank_histogram = False # ECC 상태 ecc_single_total = 0 ecc_double_total = 0 ecc_retired = 0 ecc_pending = False ecc_context = None # "single" or "double" 마지막으로 본 ECC 유형 # --- 라인 단위 파싱 (싱글 패스) --- for line in lines: stripped = line.strip() if not stripped: continue # 섹션 감지 sec_match = RE_SECTION.match(stripped) if sec_match: current_section = sec_match.group(1).strip().lower() in_dmesg = any(kw in current_section for kw in ["dmesg", "kernel", "messages"]) continue # --- NVSMI LOG: GPU PCI 헤더 및 Serial Number 항상 수집 --- gpu_sec_always = RE_GPU_SECTION.match(stripped) if gpu_sec_always: _nvsmi_cur_pci = _norm_pci(gpu_sec_always.group(1)) elif _nvsmi_cur_pci: serial_m = RE_SERIAL.search(stripped) if serial_m and "chassis" not in stripped.lower(): val = serial_m.group(1).strip() if val and val != "N/A": pci_serial_map[_nvsmi_cur_pci] = val # --- XID 에러 (모든 섹션) --- xid_m = RE_XID.search(stripped) if xid_m: ts_m = RE_TIMESTAMP.match(stripped) timestamp = ts_m.group(1) if ts_m else "" xid_events.append(XidEvent( timestamp=timestamp, xid_code=int(xid_m.group(2)), pci_address=xid_m.group(1).strip(), raw_line=stripped[:500], )) continue # --- 드라이버/CUDA 버전 (최초 발견만) --- if not driver_version: dm = RE_DRIVER.search(stripped) if dm: driver_version = dm.group(1) if not cuda_version: cm = RE_CUDA.search(stripped) if cm: cuda_version = cm.group(1) # --- nvidia-smi 테이블 행 --- smi_m = RE_SMI_ROW.search(stripped) if smi_m: smi_gpus_found = True gpu_list.append(GpuInfo( index=int(smi_m.group(1)), name=smi_m.group(2).strip(), temperature=int(smi_m.group(3)), power_draw=float(smi_m.group(4)), power_limit=float(smi_m.group(5)), memory_used=int(smi_m.group(6)), memory_total=int(smi_m.group(7)), gpu_utilization=int(smi_m.group(8)), )) continue # --- nvidia-smi -q 상세 형식 (GPU 섹션 기반) --- if not smi_gpus_found: # GPU 섹션 헤더 감지: "GPU 00000000:19:00.0" 또는 "GPU 0:" gpu_sec_m = RE_GPU_SECTION.match(stripped) or RE_GPU_SECTION_IDX.match(stripped) if gpu_sec_m: if current_gpu and current_gpu.name != "N/A": gpu_list.append(current_gpu) current_gpu = GpuInfo(index=len(gpu_list)) # PCI 주소가 헤더에 포함된 경우 addr = gpu_sec_m.group(1) if ":" in addr and "." in addr: current_gpu.pci_address = addr continue if current_gpu is not None: # 제품 정보 m = RE_GPU_NAME.search(stripped) if m: current_gpu.name = m.group(1).strip() continue m = RE_GPU_BRAND.search(stripped) if m: current_gpu.product_brand = m.group(1).strip() continue m = RE_GPU_ARCH.search(stripped) if m: current_gpu.product_architecture = m.group(1).strip() continue m = RE_UUID.search(stripped) if m: current_gpu.gpu_uuid = m.group(1).strip() continue m = RE_SERIAL.search(stripped) if m and "chassis" not in stripped.lower(): val = m.group(1).strip() if val != "N/A": current_gpu.serial_number = val continue m = RE_VBIOS.search(stripped) if m: current_gpu.vbios_version = m.group(1).strip() continue m = RE_INFOROM.search(stripped) if m: current_gpu.inforom_version = m.group(1).strip() continue # 온도 m = RE_TEMP.search(stripped) if m: current_gpu.temperature = int(m.group(1)) continue m = RE_TEMP_LIMIT.search(stripped) if m: current_gpu.temperature_limit = m.group(1).strip() continue # 전력 m = RE_POWER_DRAW.search(stripped) if m: current_gpu.power_draw = float(m.group(1)) continue m = RE_POWER_LIMIT.search(stripped) if m: current_gpu.power_limit = float(m.group(1)) continue m = RE_POWER_STATE.search(stripped) if m: current_gpu.power_state = m.group(1).strip() continue # 메모리 (FB Memory Usage 값만 유지, 이후 Conf Compute 등에 덮어쓰기 방지) m = RE_MEM_TOTAL.search(stripped) if m: val = int(m.group(1)) if current_gpu.memory_total == 0 and val > 0: current_gpu.memory_total = val continue m = RE_MEM_USED.search(stripped) if m: val = int(m.group(1)) if current_gpu.memory_used == 0: current_gpu.memory_used = val continue m = RE_MEM_FREE.search(stripped) if m: val = int(m.group(1)) if current_gpu.memory_free == 0 and val > 0: current_gpu.memory_free = val continue # 사용률 m = RE_GPU_UTIL.search(stripped) if m: current_gpu.gpu_utilization = int(m.group(1)) continue m = RE_MEM_UTIL.search(stripped) if m: current_gpu.memory_utilization = int(m.group(1)) continue # PCI m = RE_PCI.search(stripped) if m: current_gpu.pci_address = m.group(1).strip() continue m = RE_PCI_GEN.search(stripped) if m: current_gpu.pci_link_gen_current = f"Gen{m.group(1)}" continue m = RE_PCI_WIDTH.search(stripped) if m: current_gpu.pci_link_width_current = m.group(1) continue # 기타 m = RE_FAN.search(stripped) if m: current_gpu.fan_speed = int(m.group(1)) continue m = RE_PERSIST.search(stripped) if m: current_gpu.persistence_mode = m.group(1).strip() continue m = RE_COMPUTE_MODE.search(stripped) if m: current_gpu.compute_mode = m.group(1).strip() continue m = RE_MIG_MODE.search(stripped) if m: current_gpu.mig_mode = m.group(1).strip() continue m = RE_ECC_MODE.search(stripped) if m: current_gpu.ecc_mode = m.group(1).strip() continue m = RE_CLOCK_GRAPHICS.search(stripped) if m: current_gpu.clock_graphics = f"{m.group(1)} MHz" continue # HW Slowdown (순서 중요: 더 구체적인 패턴 먼저) m = RE_HW_THERMAL_SLOWDOWN.search(stripped) if m: current_gpu.hw_thermal_slowdown = m.group(1) continue m = RE_HW_POWER_BRAKE_SLOWDOWN.search(stripped) if m: current_gpu.hw_power_brake_slowdown = m.group(1) continue m = RE_HW_SLOWDOWN.search(stripped) if m: current_gpu.hw_slowdown = m.group(1) continue # --- RMA: ECC 섹션 파싱 (GPU 내부) --- if RE_ECC_SECTION.match(stripped): in_ecc_section = True ecc_subsection = None continue if in_ecc_section: if RE_VOLATILE.match(stripped): ecc_subsection = "volatile" continue if RE_AGGREGATE_SECTION.match(stripped): ecc_subsection = "aggregate" continue if RE_AGG_SRAM_SOURCES.search(stripped): ecc_subsection = "agg_sources" continue if ecc_subsection == "volatile": m = RE_SRAM_CORR.search(stripped) if m: current_gpu.sram_corr_volatile = m.group(1); continue m = RE_SRAM_UNCORR_PARITY.search(stripped) if m: current_gpu.sram_uncorr_parity_volatile = m.group(1); continue m = RE_SRAM_UNCORR_SECDED.search(stripped) if m: current_gpu.sram_uncorr_secded_volatile = m.group(1); continue m = RE_DRAM_CORR.search(stripped) if m: current_gpu.dram_corr_volatile = m.group(1); continue m = RE_DRAM_UNCORR.search(stripped) if m: current_gpu.dram_uncorr_volatile = m.group(1); continue elif ecc_subsection == "aggregate": m = RE_SRAM_CORR.search(stripped) if m: current_gpu.sram_corr_aggregate = m.group(1); continue m = RE_SRAM_UNCORR_PARITY.search(stripped) if m: current_gpu.sram_uncorr_parity_aggregate = m.group(1); continue m = RE_SRAM_UNCORR_SECDED.search(stripped) if m: current_gpu.sram_uncorr_secded_aggregate = m.group(1); continue m = RE_DRAM_CORR.search(stripped) if m: current_gpu.dram_corr_aggregate = m.group(1); continue m = RE_DRAM_UNCORR.search(stripped) if m: current_gpu.dram_uncorr_aggregate = m.group(1); continue m = RE_SRAM_THRESHOLD.search(stripped) if m: current_gpu.sram_threshold_exceeded = m.group(1); continue elif ecc_subsection == "agg_sources": m = RE_SRAM_L2.search(stripped) if m: current_gpu.sram_l2 = m.group(1); continue m = RE_SRAM_SM.search(stripped) if m: current_gpu.sram_sm = m.group(1); continue m = RE_SRAM_MC.search(stripped) if m: current_gpu.sram_microcontroller = m.group(1); continue m = RE_SRAM_PCIE.search(stripped) if m: current_gpu.sram_pcie = m.group(1); continue m = RE_SRAM_OTHER.search(stripped) if m: current_gpu.sram_other = m.group(1); continue # --- RMA: Remapped Rows 섹션 파싱 (GPU 내부) --- if RE_REMAP_SECTION.match(stripped): in_remap_section = True in_ecc_section = False in_bank_histogram = False continue if in_remap_section: m = RE_REMAP_CORR.search(stripped) if m and "remap" not in stripped.lower(): current_gpu.remap_corr_error = int(m.group(1)); continue m = RE_REMAP_UNCORR.search(stripped) if m: current_gpu.remap_uncorr_error = int(m.group(1)); continue m = RE_REMAP_FAILURE.search(stripped) if m: current_gpu.remap_failure = m.group(1); continue m = RE_REMAP_PENDING.search(stripped) if m and "page" not in stripped.lower(): current_gpu.remap_pending = m.group(1); continue if "Bank Remap Availability" in stripped: in_bank_histogram = True; continue if in_bank_histogram: m = RE_BANK_MAX.search(stripped) if m: current_gpu.remap_bank_max = int(m.group(1)); continue m = RE_BANK_HIGH.search(stripped) if m: current_gpu.remap_bank_high = int(m.group(1)); continue m = RE_BANK_PARTIAL.search(stripped) if m: current_gpu.remap_bank_partial = int(m.group(1)); continue m = RE_BANK_LOW.search(stripped) if m: current_gpu.remap_bank_low = int(m.group(1)); continue m = RE_BANK_NONE.search(stripped) if m: current_gpu.remap_bank_none = int(m.group(1)) in_remap_section = False in_bank_histogram = False continue # --- ECC 에러 전체 집계 (라인 단위 상태 머신) --- if RE_SINGLE_BIT.search(stripped) and "sram" not in stripped.lower(): ecc_context = "single" continue if RE_DOUBLE_BIT.search(stripped) and "sram" not in stripped.lower(): ecc_context = "double" continue if ecc_context: agg_m = RE_AGGREGATE.search(stripped) if agg_m: val = int(agg_m.group(1)) if ecc_context == "single": ecc_single_total = max(ecc_single_total, val) else: ecc_double_total = max(ecc_double_total, val) ecc_context = None continue ret_m = RE_RETIRED.search(stripped) if ret_m: ecc_retired = max(ecc_retired, int(ret_m.group(1))) continue if RE_PENDING.search(stripped) and "remap" not in stripped.lower(): ecc_pending = True continue # --- 커널 이벤트 (dmesg 섹션) --- if in_dmesg and stripped not in seen_kernel: if KERNEL_SKIP_KW.search(stripped): continue level = None keyword = "" # 우선순위: oom > critical > pcie > error m = KERNEL_OOM_KW.search(stripped) if m: level, keyword = "oom", m.group(1) if not level: m = KERNEL_CRITICAL_KW.search(stripped) if m: level, keyword = "critical", m.group(1) if not level: m = KERNEL_PCIE_KW.search(stripped) if m: level, keyword = "pcie_error", m.group(1) if not level: m = KERNEL_ERROR_KW.search(stripped) if m: level, keyword = "error", m.group(1) if level: seen_kernel.add(stripped) ts_m = RE_TIMESTAMP.match(stripped) kernel_events.append(KernelEvent( timestamp=ts_m.group(1) if ts_m else "", level=level, message=keyword, raw_line=stripped[:500], )) # --- 마지막 GPU 저장 --- if current_gpu and current_gpu.name != "N/A" and not smi_gpus_found: gpu_list.append(current_gpu) # --- 결과 조립 --- result.driver_version = driver_version or "N/A" result.cuda_version = cuda_version or "N/A" for gpu in gpu_list: gpu.driver_version = result.driver_version gpu.cuda_version = result.cuda_version result.gpus = gpu_list # gpu_list 시리얼 보완: NVSMI LOG에서 수집한 pci_serial_map 활용 for gpu in gpu_list: if (not gpu.serial_number or gpu.serial_number == "N/A") and gpu.pci_address: s = pci_serial_map.get(_norm_pci(gpu.pci_address)) if s: gpu.serial_number = s # PCI 주소 → GPU 매핑 (정규화 키 사용) pci_to_gpu = {} for gpu in gpu_list: if gpu.pci_address and gpu.pci_address != "N/A": pci_to_gpu[_norm_pci(gpu.pci_address)] = gpu for event in xid_events: key = _norm_pci(event.pci_address) matched = pci_to_gpu.get(key) if matched: event.gpu_index = matched.index event.gpu_name = matched.name event.gpu_serial = matched.serial_number or pci_serial_map.get(key, "") else: # gpu_list에 없어도 pci_serial_map에서 시리얼 보완 event.gpu_serial = pci_serial_map.get(key, "") result.xid_events = xid_events xid_counter = Counter(e.xid_code for e in xid_events) result.xid_summary = { code: { "count": count, "severity": XID_DB["xid_errors"].get(str(code), {}).get("severity", "warning"), "name": XID_DB["xid_errors"].get(str(code), {}).get("name", f"Unknown XID {code}"), "gpus": sorted({ f"GPU {e.gpu_index} {e.gpu_name} [S/N: {e.gpu_serial}]" if e.gpu_serial else (f"GPU {e.gpu_index} ({e.gpu_name})" if e.gpu_name else e.pci_address) for e in xid_events if e.xid_code == code }), } for code, count in xid_counter.most_common() } result.kernel_events = kernel_events kernel_counter = Counter(e.level for e in kernel_events) result.kernel_summary = dict(kernel_counter) result.ecc_info = { "single_bit_total": ecc_single_total, "double_bit_total": ecc_double_total, "retired_pages": ecc_retired, "pending_retirement": ecc_pending, } # --- RMA 판정 --- result.rma_verdicts = [evaluate_rma(gpu) for gpu in gpu_list] result.rma_count = sum(1 for v in result.rma_verdicts if v.qualifies_for_rma) # 상태 판정 critical_xids = sum(1 for e in xid_events if e.severity == "critical") warning_xids = sum(1 for e in xid_events if e.severity == "warning") result.critical_count = ( critical_xids + kernel_counter.get("critical", 0) + kernel_counter.get("oom", 0) + (1 if ecc_double_total > 0 else 0) + result.rma_count # RMA 대상 GPU 수도 critical에 포함 ) result.warning_count = ( warning_xids + kernel_counter.get("error", 0) + kernel_counter.get("pcie_error", 0) + (1 if ecc_single_total > 10 else 0) ) result.info_count = sum(1 for e in xid_events if e.severity == "info") result.total_issues = result.critical_count + result.warning_count if result.rma_count > 0: result.overall_status = "critical" elif result.critical_count > 0: result.overall_status = "critical" elif result.warning_count > 0: result.overall_status = "warning" else: result.overall_status = "healthy" return result def _safe_int(val: str) -> int: """N/A 등 숫자가 아닌 값을 0으로 변환""" try: return int(val) except (ValueError, TypeError): return 0 def evaluate_rma(gpu: GpuInfo) -> RmaVerdict: """ NVIDIA Hopper RMA 기준에 따라 GPU의 RMA 적격 여부를 판정합니다. [Row-Remapping RMA 기준] 1. Remapping Failure Occurred = Yes → 즉시 RMA 2. 한 Bank에서 Uncorrectable Error Row 8회 이상 리매핑 3. 총 512회 리매핑 발생 4. 리매핑 8회 후 Field Diagnostics에서 문제 감지 [SRAM RMA 기준] 1. SRAM Threshold Exceeded = Yes → 즉시 RMA 2. Parity SRAM: 주소 뱅크 내 4개 이상 UCE Unique Count 3. SECDED ECC SRAM: 주소 뱅크 내 2개 이상 UCE Unique Count """ verdict = RmaVerdict( gpu_index=gpu.index, gpu_name=gpu.name, serial_number=gpu.serial_number, ) # --- Row-Remapping 분석 --- total_remaps = gpu.remap_corr_error + gpu.remap_uncorr_error verdict.row_remap_details = { "correctable_error": gpu.remap_corr_error, "uncorrectable_error": gpu.remap_uncorr_error, "total_remaps": total_remaps, "pending": gpu.remap_pending, "failure_occurred": gpu.remap_failure, "bank_max": gpu.remap_bank_max, "bank_high": gpu.remap_bank_high, "bank_partial": gpu.remap_bank_partial, "bank_low": gpu.remap_bank_low, "bank_none": gpu.remap_bank_none, } # RMA 기준 1: Remapping Failure Occurred = Yes if gpu.remap_failure.lower() == "yes": verdict.qualifies_for_rma = True verdict.reasons.append( "Remapping Failure Occurred = Yes: " "리매핑 실패 플래그 설정됨. 즉시 RMA 대상." ) # RMA 기준 2: Uncorrectable Error Row 8회 이상 (Bank당) if gpu.remap_uncorr_error >= 8: verdict.qualifies_for_rma = True verdict.reasons.append( f"Uncorrectable Error 리매핑 {gpu.remap_uncorr_error}회: " f"Bank당 8회 이상 Uncorrectable Error 리매핑 발생 (기준: 8회)." ) # RMA 기준 3: 총 512회 리매핑 if total_remaps >= 512: verdict.qualifies_for_rma = True verdict.reasons.append( f"총 리매핑 {total_remaps}회: " f"Uncorrectable Memory Error로 총 512회 이상 리매핑 발생 (기준: 512회)." ) # 경고: 리매핑 진행 중 if gpu.remap_pending.lower() == "yes": verdict.warnings.append( "Row Remapping Pending = Yes: 리매핑 대기 중. 재부팅 후 적용됩니다." ) # 경고: Bank 가용성 낮음 if gpu.remap_bank_none > 0: verdict.warnings.append( f"Bank Remap None = {gpu.remap_bank_none}: " f"리매핑 가용 뱅크가 소진된 뱅크가 있습니다." ) if gpu.remap_bank_low > 0: verdict.warnings.append( f"Bank Remap Low = {gpu.remap_bank_low}: " f"리매핑 가용 뱅크가 적은 뱅크가 있습니다." ) # 경고: 리매핑이 상당수 발생 if 0 < gpu.remap_uncorr_error < 8: verdict.warnings.append( f"Uncorrectable Error 리매핑 {gpu.remap_uncorr_error}회: " f"아직 RMA 기준(8회) 미달이나 모니터링 필요." ) # --- SRAM 분석 --- sram_uncorr_parity = _safe_int(gpu.sram_uncorr_parity_aggregate) sram_uncorr_secded = _safe_int(gpu.sram_uncorr_secded_aggregate) verdict.sram_details = { "volatile": { "correctable": gpu.sram_corr_volatile, "uncorrectable_parity": gpu.sram_uncorr_parity_volatile, "uncorrectable_secded": gpu.sram_uncorr_secded_volatile, }, "aggregate": { "correctable": gpu.sram_corr_aggregate, "uncorrectable_parity": gpu.sram_uncorr_parity_aggregate, "uncorrectable_secded": gpu.sram_uncorr_secded_aggregate, "threshold_exceeded": gpu.sram_threshold_exceeded, }, "sources": { "l2": gpu.sram_l2, "sm": gpu.sram_sm, "microcontroller": gpu.sram_microcontroller, "pcie": gpu.sram_pcie, "other": gpu.sram_other, }, "dram": { "corr_volatile": gpu.dram_corr_volatile, "uncorr_volatile": gpu.dram_uncorr_volatile, "corr_aggregate": gpu.dram_corr_aggregate, "uncorr_aggregate": gpu.dram_uncorr_aggregate, }, } # SRAM RMA 기준 1: Threshold Exceeded = Yes if gpu.sram_threshold_exceeded.lower() == "yes": verdict.qualifies_for_rma = True verdict.reasons.append( "SRAM Threshold Exceeded = Yes: " "SRAM UCE 임계치 초과 플래그 설정됨. 즉시 RMA 대상." ) # SRAM RMA 기준 2: Parity SRAM UCE >= 4 if sram_uncorr_parity >= 4: verdict.qualifies_for_rma = True verdict.reasons.append( f"SRAM Uncorrectable Parity (Aggregate) = {sram_uncorr_parity}: " f"Parity 보호 SRAM에서 4개 이상 UCE 발생 (기준: 4개)." ) # SRAM RMA 기준 3: SECDED ECC SRAM UCE >= 2 if sram_uncorr_secded >= 2: verdict.qualifies_for_rma = True verdict.reasons.append( f"SRAM Uncorrectable SEC-DED (Aggregate) = {sram_uncorr_secded}: " f"SECDED ECC 보호 SRAM에서 2개 이상 UCE 발생 (기준: 2개)." ) # 경고: SRAM UCE가 있지만 기준 미달 if 0 < sram_uncorr_parity < 4: verdict.warnings.append( f"SRAM Parity UCE = {sram_uncorr_parity}: " f"아직 RMA 기준(4개) 미달이나 모니터링 필요." ) if 0 < sram_uncorr_secded < 2: verdict.warnings.append( f"SRAM SECDED UCE = {sram_uncorr_secded}: " f"아직 RMA 기준(2개) 미달이나 모니터링 필요." ) return verdict