unknown
1004 lines
40 KiB
Python
1004 lines
40 KiB
Python
"""
|
|
NV-Log Analyzer - 핵심 분석 엔진
|
|
nvidia-bug-report.log.gz 파일을 파싱하여 XID 에러, GPU 상태, 커널 로그를 분석합니다.
|
|
성능 최적화: 모든 파싱을 라인 단위로 처리하여 대용량 파일에서도 빠르게 동작합니다.
|
|
"""
|
|
|
|
import gzip
|
|
import json
|
|
import re
|
|
from collections import Counter, defaultdict
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
|
|
# XID 데이터베이스 로드
|
|
DATA_DIR = Path(__file__).parent / "data"
|
|
with open(DATA_DIR / "xid_database.json", "r", encoding="utf-8") as f:
|
|
XID_DB = json.load(f)
|
|
|
|
|
|
# --- 데이터 클래스 ---
|
|
|
|
@dataclass
|
|
class XidEvent:
|
|
timestamp: str
|
|
xid_code: int
|
|
pci_address: str
|
|
raw_line: str
|
|
name: str = ""
|
|
severity: str = "info"
|
|
description: str = ""
|
|
action: str = ""
|
|
action_ko: str = ""
|
|
action_en: str = ""
|
|
gpu_index: int = -1
|
|
gpu_name: str = ""
|
|
gpu_serial: str = ""
|
|
|
|
def __post_init__(self):
|
|
xid_info = XID_DB["xid_errors"].get(str(self.xid_code))
|
|
if xid_info:
|
|
self.name = xid_info["name"]
|
|
self.severity = xid_info["severity"]
|
|
self.description = xid_info["description"]
|
|
self.action = xid_info.get("action_ko", xid_info.get("action", ""))
|
|
self.action_ko = xid_info.get("action_ko", xid_info.get("action", ""))
|
|
self.action_en = xid_info.get("action_en", "NVIDIA official documentation does not specify a detailed action for this code.")
|
|
else:
|
|
self.name = f"Unknown XID {self.xid_code}"
|
|
self.severity = "warning"
|
|
self.description = f"XID {self.xid_code}에 대한 정보가 데이터베이스에 없습니다."
|
|
self.action = "NVIDIA 공식 문서에서 해당 XID 코드를 확인하세요."
|
|
self.action_ko = "NVIDIA 공식 문서에서 해당 XID 코드를 확인하세요."
|
|
self.action_en = "Please check this XID code in NVIDIA official documentation."
|
|
|
|
|
|
@dataclass
|
|
class GpuInfo:
|
|
index: int = 0
|
|
name: str = "N/A"
|
|
product_brand: str = "N/A"
|
|
product_architecture: str = "N/A"
|
|
driver_version: str = "N/A"
|
|
cuda_version: str = "N/A"
|
|
vbios_version: str = "N/A"
|
|
inforom_version: str = "N/A"
|
|
gpu_uuid: str = "N/A"
|
|
temperature: int = 0
|
|
temperature_limit: str = "N/A"
|
|
power_draw: float = 0.0
|
|
power_limit: float = 0.0
|
|
power_state: str = "N/A"
|
|
memory_used: int = 0
|
|
memory_total: int = 0
|
|
memory_free: int = 0
|
|
bar1_total: int = 0
|
|
bar1_used: int = 0
|
|
gpu_utilization: int = 0
|
|
memory_utilization: int = 0
|
|
ecc_mode: str = "N/A"
|
|
ecc_single_bit: int = 0
|
|
ecc_double_bit: int = 0
|
|
retired_pages_single: int = 0
|
|
retired_pages_double: int = 0
|
|
pci_address: str = "N/A"
|
|
pci_link_gen_current: str = "N/A"
|
|
pci_link_width_current: str = "N/A"
|
|
serial_number: str = "N/A"
|
|
fan_speed: int = 0
|
|
persistence_mode: str = "N/A"
|
|
compute_mode: str = "N/A"
|
|
mig_mode: str = "N/A"
|
|
clock_graphics: str = "N/A"
|
|
clock_memory: str = "N/A"
|
|
clock_max_graphics: str = "N/A"
|
|
clock_max_memory: str = "N/A"
|
|
# --- HW Slowdown ---
|
|
hw_slowdown: str = "N/A"
|
|
hw_thermal_slowdown: str = "N/A"
|
|
hw_power_brake_slowdown: str = "N/A"
|
|
# --- RMA 관련 필드 ---
|
|
# Row Remapping
|
|
remap_corr_error: int = 0
|
|
remap_uncorr_error: int = 0
|
|
remap_pending: str = "N/A"
|
|
remap_failure: str = "N/A"
|
|
remap_bank_max: int = 0
|
|
remap_bank_high: int = 0
|
|
remap_bank_partial: int = 0
|
|
remap_bank_low: int = 0
|
|
remap_bank_none: int = 0
|
|
# SRAM ECC
|
|
sram_corr_volatile: str = "N/A"
|
|
sram_uncorr_parity_volatile: str = "N/A"
|
|
sram_uncorr_secded_volatile: str = "N/A"
|
|
sram_corr_aggregate: str = "N/A"
|
|
sram_uncorr_parity_aggregate: str = "N/A"
|
|
sram_uncorr_secded_aggregate: str = "N/A"
|
|
sram_threshold_exceeded: str = "N/A"
|
|
# SRAM Sources
|
|
sram_l2: str = "N/A"
|
|
sram_sm: str = "N/A"
|
|
sram_microcontroller: str = "N/A"
|
|
sram_pcie: str = "N/A"
|
|
sram_other: str = "N/A"
|
|
# DRAM ECC
|
|
dram_corr_volatile: str = "N/A"
|
|
dram_uncorr_volatile: str = "N/A"
|
|
dram_corr_aggregate: str = "N/A"
|
|
dram_uncorr_aggregate: str = "N/A"
|
|
|
|
|
|
@dataclass
|
|
class RmaVerdict:
|
|
"""GPU별 RMA 판정 결과"""
|
|
gpu_index: int = 0
|
|
gpu_name: str = ""
|
|
serial_number: str = ""
|
|
qualifies_for_rma: bool = False
|
|
reasons: list = field(default_factory=list)
|
|
warnings: list = field(default_factory=list)
|
|
row_remap_details: dict = field(default_factory=dict)
|
|
sram_details: dict = field(default_factory=dict)
|
|
|
|
|
|
@dataclass
|
|
class KernelEvent:
|
|
timestamp: str
|
|
level: str
|
|
message: str
|
|
raw_line: str
|
|
|
|
|
|
@dataclass
|
|
class AnalysisResult:
|
|
filename: str = ""
|
|
file_size_mb: float = 0.0
|
|
analysis_time: str = ""
|
|
gpus: list = field(default_factory=list)
|
|
driver_version: str = "N/A"
|
|
cuda_version: str = "N/A"
|
|
xid_events: list = field(default_factory=list)
|
|
xid_summary: dict = field(default_factory=dict)
|
|
kernel_events: list = field(default_factory=list)
|
|
kernel_summary: dict = field(default_factory=dict)
|
|
ecc_info: dict = field(default_factory=dict)
|
|
rma_verdicts: list = field(default_factory=list) # List[RmaVerdict]
|
|
rma_count: int = 0 # RMA 대상 GPU 수
|
|
overall_status: str = "healthy"
|
|
total_issues: int = 0
|
|
critical_count: int = 0
|
|
warning_count: int = 0
|
|
info_count: int = 0
|
|
|
|
|
|
# --- 라인 단위 정규표현식 (성능 최적화) ---
|
|
|
|
# XID: 한 라인 안에서만 매칭
|
|
RE_XID = re.compile(r"NVRM:\s*Xid\s*\(PCI:([^)]+)\):\s*(\d+)")
|
|
RE_TIMESTAMP = re.compile(r"^(\w{3}\s+\d+\s+[\d:]+|\d{4}-\d{2}-\d{2}[T ][\d:]+|[\d]+\.[\d]+)")
|
|
|
|
# GPU 정보 (라인 단위) - nvidia-smi -q 출력의 "Key : Value" 패턴
|
|
RE_GPU_NAME = re.compile(r"Product Name\s*:\s*(.+)", re.IGNORECASE)
|
|
RE_GPU_BRAND = re.compile(r"Product Brand\s*:\s*(.+)", re.IGNORECASE)
|
|
RE_GPU_ARCH = re.compile(r"Product Architecture\s*:\s*(.+)", re.IGNORECASE)
|
|
RE_DRIVER = re.compile(r"Driver Version\s*:\s*([\d.]+)", re.IGNORECASE)
|
|
RE_CUDA = re.compile(r"CUDA Version\s*:\s*([\d.]+)", re.IGNORECASE)
|
|
RE_VBIOS = re.compile(r"VBIOS Version\s*:\s*(\S+)", re.IGNORECASE)
|
|
RE_INFOROM = re.compile(r"Image Version\s*:\s*(\S+)", re.IGNORECASE)
|
|
RE_UUID = re.compile(r"GPU UUID\s*:\s*(GPU-\S+)", re.IGNORECASE)
|
|
RE_TEMP = re.compile(r"GPU Current Temp\s*:\s*(\d+)\s*C", re.IGNORECASE)
|
|
RE_TEMP_LIMIT = re.compile(r"GPU T\.?Limit Temp\s*:\s*(.+)", re.IGNORECASE)
|
|
RE_POWER_DRAW = re.compile(r"Power Draw\s*:\s*([\d.]+)\s*W", re.IGNORECASE)
|
|
RE_POWER_LIMIT = re.compile(r"(?:Default |Enforced |Current )?Power Limit\s*:\s*([\d.]+)\s*W", re.IGNORECASE)
|
|
RE_POWER_STATE = re.compile(r"Power State\s*:\s*(\S+)", re.IGNORECASE)
|
|
RE_MEM_USED = re.compile(r"Used\s*:\s*(\d+)\s*MiB", re.IGNORECASE)
|
|
RE_MEM_TOTAL = re.compile(r"Total\s*:\s*(\d+)\s*MiB", re.IGNORECASE)
|
|
RE_MEM_FREE = re.compile(r"Free\s*:\s*(\d+)\s*MiB", re.IGNORECASE)
|
|
RE_GPU_UTIL = re.compile(r"Gpu\s*:\s*(\d+)\s*%", re.IGNORECASE)
|
|
RE_MEM_UTIL = re.compile(r"Memory\s*:\s*(\d+)\s*%", re.IGNORECASE)
|
|
RE_PCI = re.compile(r"Bus Id\s*:\s*([\dA-Fa-f:.]+)", re.IGNORECASE)
|
|
RE_PCI_GEN = re.compile(r"Current\s*:\s*Gen(\d+)", re.IGNORECASE)
|
|
RE_PCI_WIDTH = re.compile(r"Current\s*:\s*(\d+x)", re.IGNORECASE)
|
|
RE_SERIAL = re.compile(r"^\s*Serial Number\s*:\s*(\S+)", re.IGNORECASE)
|
|
RE_FAN = re.compile(r"Fan Speed\s*:\s*(\d+)\s*%", re.IGNORECASE)
|
|
RE_PERSIST = re.compile(r"Persistence Mode\s*:\s*(\w+)", re.IGNORECASE)
|
|
RE_COMPUTE_MODE = re.compile(r"Compute Mode\s*:\s*(.+)", re.IGNORECASE)
|
|
RE_MIG_MODE = re.compile(r"MIG Mode\s*:\s*(\w+)", re.IGNORECASE)
|
|
RE_ECC_MODE = re.compile(r"ECC Mode[\s\S]*?Current\s*:\s*(\w+)", re.IGNORECASE)
|
|
RE_CLOCK_GRAPHICS = re.compile(r"Graphics\s*:\s*(\d+)\s*MHz", re.IGNORECASE)
|
|
RE_CLOCK_MEMORY = re.compile(r"SM\s*:\s*(\d+)\s*MHz|Memory\s*:\s*(\d+)\s*MHz", re.IGNORECASE)
|
|
RE_HW_SLOWDOWN = re.compile(r"HW Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE)
|
|
RE_HW_THERMAL_SLOWDOWN = re.compile(r"HW Thermal Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE)
|
|
RE_HW_POWER_BRAKE_SLOWDOWN = re.compile(r"HW Power Brake Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE)
|
|
# GPU 섹션 헤더: "GPU 00000000:19:00.0" 또는 "GPU 0000:01:00.0" 또는 "GPU 0:"
|
|
RE_GPU_SECTION = re.compile(r"^GPU\s+([\dA-Fa-f]{4,8}:[\dA-Fa-f]{2}:[\dA-Fa-f]{2}\.\d)\s*$", re.IGNORECASE)
|
|
RE_GPU_SECTION_IDX = re.compile(r"^GPU\s+(\d+)\s*:\s*$", re.IGNORECASE)
|
|
|
|
# nvidia-smi 테이블 행
|
|
RE_SMI_ROW = re.compile(
|
|
r"\|\s*(\d+)\s+(.+?)\s+\w+\s*\|.*?(\d+)C.*?([\d.]+)W\s*/\s*([\d.]+)W.*?(\d+)MiB\s*/\s*(\d+)MiB.*?(\d+)%"
|
|
)
|
|
|
|
# ECC (라인 단위)
|
|
RE_SINGLE_BIT = re.compile(r"Single Bit", re.IGNORECASE)
|
|
RE_DOUBLE_BIT = re.compile(r"Double Bit", re.IGNORECASE)
|
|
RE_AGGREGATE = re.compile(r"Aggregate\s*:\s*(\d+)", re.IGNORECASE)
|
|
RE_RETIRED = re.compile(r"Retired pages\s*:\s*(\d+)", re.IGNORECASE)
|
|
RE_PENDING = re.compile(r"Pending\s*.*:\s*Yes", re.IGNORECASE)
|
|
|
|
# --- RMA 관련 정규표현식 ---
|
|
# Row Remapping
|
|
RE_REMAP_SECTION = re.compile(r"Remapped Rows", re.IGNORECASE)
|
|
RE_REMAP_CORR = re.compile(r"Correctable Error\s*:\s*(\d+)", re.IGNORECASE)
|
|
RE_REMAP_UNCORR = re.compile(r"Uncorrectable Error\s*:\s*(\d+)", re.IGNORECASE)
|
|
RE_REMAP_PENDING = re.compile(r"Pending\s*:\s*(Yes|No)", re.IGNORECASE)
|
|
RE_REMAP_FAILURE = re.compile(r"Remapping Failure Occurred\s*:\s*(Yes|No)", re.IGNORECASE)
|
|
RE_BANK_MAX = re.compile(r"Max\s*:\s*(\d+)\s*bank", re.IGNORECASE)
|
|
RE_BANK_HIGH = re.compile(r"High\s*:\s*(\d+)\s*bank", re.IGNORECASE)
|
|
RE_BANK_PARTIAL = re.compile(r"Partial\s*:\s*(\d+)\s*bank", re.IGNORECASE)
|
|
RE_BANK_LOW = re.compile(r"Low\s*:\s*(\d+)\s*bank", re.IGNORECASE)
|
|
RE_BANK_NONE = re.compile(r"None\s*:\s*(\d+)\s*bank", re.IGNORECASE)
|
|
# SRAM ECC
|
|
RE_SRAM_CORR = re.compile(r"SRAM Correctable\s*:\s*(\S+)", re.IGNORECASE)
|
|
RE_SRAM_UNCORR_PARITY = re.compile(r"SRAM Uncorrectable Parity\s*:\s*(\S+)", re.IGNORECASE)
|
|
RE_SRAM_UNCORR_SECDED = re.compile(r"SRAM Uncorrectable SEC-?DED\s*:\s*(\S+)", re.IGNORECASE)
|
|
RE_SRAM_THRESHOLD = re.compile(r"SRAM Threshold Exceeded\s*:\s*(\S+)", re.IGNORECASE)
|
|
# SRAM Sources
|
|
RE_SRAM_L2 = re.compile(r"SRAM L2\s*:\s*(\S+)", re.IGNORECASE)
|
|
RE_SRAM_SM = re.compile(r"SRAM SM\s*:\s*(\S+)", re.IGNORECASE)
|
|
RE_SRAM_MC = re.compile(r"SRAM Microcontroller\s*:\s*(\S+)", re.IGNORECASE)
|
|
RE_SRAM_PCIE = re.compile(r"SRAM PCIE\s*:\s*(\S+)", re.IGNORECASE)
|
|
RE_SRAM_OTHER = re.compile(r"SRAM Other\s*:\s*(\S+)", re.IGNORECASE)
|
|
# DRAM ECC
|
|
RE_DRAM_CORR = re.compile(r"DRAM Correctable\s*:\s*(\S+)", re.IGNORECASE)
|
|
RE_DRAM_UNCORR = re.compile(r"DRAM Uncorrectable\s*:\s*(\S+)", re.IGNORECASE)
|
|
# ECC Section context
|
|
RE_ECC_SECTION = re.compile(r"ECC Errors", re.IGNORECASE)
|
|
RE_VOLATILE = re.compile(r"^\s*Volatile\s*$", re.IGNORECASE)
|
|
RE_AGGREGATE_SECTION = re.compile(r"^\s*Aggregate\s*$", re.IGNORECASE)
|
|
RE_AGG_SRAM_SOURCES = re.compile(r"Aggregate Uncorrectable SRAM Sources", re.IGNORECASE)
|
|
|
|
# 커널 이벤트 (라인 단위 키워드 매칭)
|
|
KERNEL_CRITICAL_KW = re.compile(
|
|
r"(kernel panic|hardware error|machine check|fatal|BUG:|RIP:)", re.IGNORECASE
|
|
)
|
|
KERNEL_ERROR_KW = re.compile(
|
|
r"(error|failed|failure|segfault|cannot|unable to)", re.IGNORECASE
|
|
)
|
|
KERNEL_OOM_KW = re.compile(
|
|
r"(Out of memory|oom-killer|invoked oom|Killed process)", re.IGNORECASE
|
|
)
|
|
KERNEL_PCIE_KW = re.compile(
|
|
r"(AER|correctable error|uncorrectable error)", re.IGNORECASE
|
|
)
|
|
KERNEL_SKIP_KW = re.compile(r"(audit|systemd|session|polkit|dbus)", re.IGNORECASE)
|
|
|
|
# 섹션 구분: 언더스코어(_) 또는 대시(-)가 양옆에 있고 대괄호([, ])가 있을 수도 없을 수도 있는 구분선 매칭
|
|
# nvidia-bug-report.log 내 dmesg 등 다양한 로그 섹션 구분선을 감지합니다.
|
|
RE_SECTION = re.compile(r"^[_-]+\s*\[?(.+?)\]?\s*[_-]+$")
|
|
|
|
|
|
# --- 파서 함수들 ---
|
|
|
|
_RE_PCI_NORMALIZE = re.compile(
|
|
r"^([\dA-Fa-f]+):([\dA-Fa-f]{1,2}):([\dA-Fa-f]{1,2})(?:\.\d)?$"
|
|
)
|
|
_RE_PCI_NO_DOMAIN = re.compile(
|
|
r"^([\dA-Fa-f]{1,2}):([\dA-Fa-f]{1,2})(?:\.\d)?$"
|
|
)
|
|
|
|
def _norm_pci(addr: str) -> str:
|
|
"""
|
|
PCI 주소를 'DOMAIN:BUS:DEVICE' 형태로 정규화.
|
|
도메인은 정수값으로, bus/device는 소문자 2자리 hex로 통일.
|
|
- '0000:19:00' → '0:19:00'
|
|
- '00000000:19:00.0' → '0:19:00'
|
|
- '0001:3b:00.0' → '1:3b:00' (다른 도메인 → 다른 키)
|
|
- '19:00.0' → '0:19:00' (도메인 생략 → 0으로 가정)
|
|
매칭 실패 시 원문 소문자 반환.
|
|
"""
|
|
s = addr.strip()
|
|
m = _RE_PCI_NORMALIZE.match(s)
|
|
if m:
|
|
domain = int(m.group(1), 16)
|
|
bus = m.group(2).zfill(2).lower()
|
|
device = m.group(3).zfill(2).lower()
|
|
return f"{domain}:{bus}:{device}"
|
|
m2 = _RE_PCI_NO_DOMAIN.match(s)
|
|
if m2:
|
|
bus = m2.group(1).zfill(2).lower()
|
|
device = m2.group(2).zfill(2).lower()
|
|
return f"0:{bus}:{device}"
|
|
return s.lower()
|
|
|
|
|
|
def read_log_file(file_content: bytes, filename: str = "") -> str:
|
|
"""gz 또는 텍스트 파일을 읽어 문자열로 반환"""
|
|
if filename.endswith(".gz") or file_content[:2] == b'\x1f\x8b':
|
|
try:
|
|
return gzip.decompress(file_content).decode("utf-8", errors="replace")
|
|
except gzip.BadGzipFile:
|
|
return file_content.decode("utf-8", errors="replace")
|
|
return file_content.decode("utf-8", errors="replace")
|
|
|
|
|
|
def analyze_log(file_content: bytes, filename: str = "uploaded_file") -> AnalysisResult:
|
|
"""메인 분석 함수 - 라인 단위 싱글 패스 처리"""
|
|
result = AnalysisResult(
|
|
filename=filename,
|
|
file_size_mb=round(len(file_content) / (1024 * 1024), 2),
|
|
analysis_time=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
|
)
|
|
|
|
log_text = read_log_file(file_content, filename)
|
|
lines = log_text.splitlines()
|
|
|
|
# --- 상태 변수들 ---
|
|
current_section = "header"
|
|
driver_version = None
|
|
cuda_version = None
|
|
|
|
# GPU 파싱 상태
|
|
gpu_list = []
|
|
current_gpu = None
|
|
smi_gpus_found = False
|
|
|
|
# NVSMI LOG 상세 섹션의 PCI → Serial 별도 수집 (smi_gpus_found 여부 무관)
|
|
pci_serial_map: dict = {} # { "00000000:19:00.0" : "serial_string" }
|
|
_nvsmi_cur_pci: str = "" # 현재 GPU PCI (NVSMI 상세 파싱용)
|
|
|
|
# XID 이벤트
|
|
xid_events = []
|
|
|
|
# 커널 이벤트 (dmesg 섹션만)
|
|
kernel_events = []
|
|
seen_kernel = set()
|
|
in_dmesg = False
|
|
|
|
# RMA/ECC 파싱 상태 머신
|
|
in_ecc_section = False
|
|
ecc_subsection = None # "volatile", "aggregate", "agg_sources"
|
|
in_remap_section = False
|
|
in_bank_histogram = False
|
|
|
|
# ECC 상태
|
|
ecc_single_total = 0
|
|
ecc_double_total = 0
|
|
ecc_retired = 0
|
|
ecc_pending = False
|
|
ecc_context = None # "single" or "double" 마지막으로 본 ECC 유형
|
|
|
|
# --- 라인 단위 파싱 (싱글 패스) ---
|
|
for line in lines:
|
|
stripped = line.strip()
|
|
if not stripped:
|
|
continue
|
|
|
|
# 섹션 감지
|
|
sec_match = RE_SECTION.match(stripped)
|
|
if sec_match:
|
|
current_section = sec_match.group(1).strip().lower()
|
|
in_dmesg = any(kw in current_section for kw in ["dmesg", "kernel", "messages"])
|
|
continue
|
|
|
|
# --- NVSMI LOG: GPU PCI 헤더 및 Serial Number 항상 수집 ---
|
|
gpu_sec_always = RE_GPU_SECTION.match(stripped)
|
|
if gpu_sec_always:
|
|
_nvsmi_cur_pci = _norm_pci(gpu_sec_always.group(1))
|
|
elif _nvsmi_cur_pci:
|
|
serial_m = RE_SERIAL.search(stripped)
|
|
if serial_m and "chassis" not in stripped.lower():
|
|
val = serial_m.group(1).strip()
|
|
if val and val != "N/A":
|
|
pci_serial_map[_nvsmi_cur_pci] = val
|
|
|
|
# --- XID 에러 (모든 섹션) ---
|
|
xid_m = RE_XID.search(stripped)
|
|
if xid_m:
|
|
ts_m = RE_TIMESTAMP.match(stripped)
|
|
timestamp = ts_m.group(1) if ts_m else ""
|
|
xid_events.append(XidEvent(
|
|
timestamp=timestamp,
|
|
xid_code=int(xid_m.group(2)),
|
|
pci_address=xid_m.group(1).strip(),
|
|
raw_line=stripped[:500],
|
|
))
|
|
continue
|
|
|
|
# --- 드라이버/CUDA 버전 (최초 발견만) ---
|
|
if not driver_version:
|
|
dm = RE_DRIVER.search(stripped)
|
|
if dm:
|
|
driver_version = dm.group(1)
|
|
if not cuda_version:
|
|
cm = RE_CUDA.search(stripped)
|
|
if cm:
|
|
cuda_version = cm.group(1)
|
|
|
|
# --- nvidia-smi 테이블 행 ---
|
|
smi_m = RE_SMI_ROW.search(stripped)
|
|
if smi_m:
|
|
smi_gpus_found = True
|
|
gpu_list.append(GpuInfo(
|
|
index=int(smi_m.group(1)),
|
|
name=smi_m.group(2).strip(),
|
|
temperature=int(smi_m.group(3)),
|
|
power_draw=float(smi_m.group(4)),
|
|
power_limit=float(smi_m.group(5)),
|
|
memory_used=int(smi_m.group(6)),
|
|
memory_total=int(smi_m.group(7)),
|
|
gpu_utilization=int(smi_m.group(8)),
|
|
))
|
|
continue
|
|
|
|
# --- nvidia-smi -q 상세 형식 (GPU 섹션 기반) ---
|
|
if not smi_gpus_found:
|
|
# GPU 섹션 헤더 감지: "GPU 00000000:19:00.0" 또는 "GPU 0:"
|
|
gpu_sec_m = RE_GPU_SECTION.match(stripped) or RE_GPU_SECTION_IDX.match(stripped)
|
|
if gpu_sec_m:
|
|
if current_gpu and current_gpu.name != "N/A":
|
|
gpu_list.append(current_gpu)
|
|
current_gpu = GpuInfo(index=len(gpu_list))
|
|
# PCI 주소가 헤더에 포함된 경우
|
|
addr = gpu_sec_m.group(1)
|
|
if ":" in addr and "." in addr:
|
|
current_gpu.pci_address = addr
|
|
continue
|
|
|
|
if current_gpu is not None:
|
|
# 제품 정보
|
|
m = RE_GPU_NAME.search(stripped)
|
|
if m:
|
|
current_gpu.name = m.group(1).strip()
|
|
continue
|
|
m = RE_GPU_BRAND.search(stripped)
|
|
if m:
|
|
current_gpu.product_brand = m.group(1).strip()
|
|
continue
|
|
m = RE_GPU_ARCH.search(stripped)
|
|
if m:
|
|
current_gpu.product_architecture = m.group(1).strip()
|
|
continue
|
|
m = RE_UUID.search(stripped)
|
|
if m:
|
|
current_gpu.gpu_uuid = m.group(1).strip()
|
|
continue
|
|
m = RE_SERIAL.search(stripped)
|
|
if m and "chassis" not in stripped.lower():
|
|
val = m.group(1).strip()
|
|
if val != "N/A":
|
|
current_gpu.serial_number = val
|
|
continue
|
|
m = RE_VBIOS.search(stripped)
|
|
if m:
|
|
current_gpu.vbios_version = m.group(1).strip()
|
|
continue
|
|
m = RE_INFOROM.search(stripped)
|
|
if m:
|
|
current_gpu.inforom_version = m.group(1).strip()
|
|
continue
|
|
|
|
# 온도
|
|
m = RE_TEMP.search(stripped)
|
|
if m:
|
|
current_gpu.temperature = int(m.group(1))
|
|
continue
|
|
m = RE_TEMP_LIMIT.search(stripped)
|
|
if m:
|
|
current_gpu.temperature_limit = m.group(1).strip()
|
|
continue
|
|
|
|
# 전력
|
|
m = RE_POWER_DRAW.search(stripped)
|
|
if m:
|
|
current_gpu.power_draw = float(m.group(1))
|
|
continue
|
|
m = RE_POWER_LIMIT.search(stripped)
|
|
if m:
|
|
current_gpu.power_limit = float(m.group(1))
|
|
continue
|
|
m = RE_POWER_STATE.search(stripped)
|
|
if m:
|
|
current_gpu.power_state = m.group(1).strip()
|
|
continue
|
|
|
|
# 메모리 (FB Memory Usage 값만 유지, 이후 Conf Compute 등에 덮어쓰기 방지)
|
|
m = RE_MEM_TOTAL.search(stripped)
|
|
if m:
|
|
val = int(m.group(1))
|
|
if current_gpu.memory_total == 0 and val > 0:
|
|
current_gpu.memory_total = val
|
|
continue
|
|
m = RE_MEM_USED.search(stripped)
|
|
if m:
|
|
val = int(m.group(1))
|
|
if current_gpu.memory_used == 0:
|
|
current_gpu.memory_used = val
|
|
continue
|
|
m = RE_MEM_FREE.search(stripped)
|
|
if m:
|
|
val = int(m.group(1))
|
|
if current_gpu.memory_free == 0 and val > 0:
|
|
current_gpu.memory_free = val
|
|
continue
|
|
|
|
# 사용률
|
|
m = RE_GPU_UTIL.search(stripped)
|
|
if m:
|
|
current_gpu.gpu_utilization = int(m.group(1))
|
|
continue
|
|
m = RE_MEM_UTIL.search(stripped)
|
|
if m:
|
|
current_gpu.memory_utilization = int(m.group(1))
|
|
continue
|
|
|
|
# PCI
|
|
m = RE_PCI.search(stripped)
|
|
if m:
|
|
current_gpu.pci_address = m.group(1).strip()
|
|
continue
|
|
m = RE_PCI_GEN.search(stripped)
|
|
if m:
|
|
current_gpu.pci_link_gen_current = f"Gen{m.group(1)}"
|
|
continue
|
|
m = RE_PCI_WIDTH.search(stripped)
|
|
if m:
|
|
current_gpu.pci_link_width_current = m.group(1)
|
|
continue
|
|
|
|
# 기타
|
|
m = RE_FAN.search(stripped)
|
|
if m:
|
|
current_gpu.fan_speed = int(m.group(1))
|
|
continue
|
|
m = RE_PERSIST.search(stripped)
|
|
if m:
|
|
current_gpu.persistence_mode = m.group(1).strip()
|
|
continue
|
|
m = RE_COMPUTE_MODE.search(stripped)
|
|
if m:
|
|
current_gpu.compute_mode = m.group(1).strip()
|
|
continue
|
|
m = RE_MIG_MODE.search(stripped)
|
|
if m:
|
|
current_gpu.mig_mode = m.group(1).strip()
|
|
continue
|
|
m = RE_ECC_MODE.search(stripped)
|
|
if m:
|
|
current_gpu.ecc_mode = m.group(1).strip()
|
|
continue
|
|
m = RE_CLOCK_GRAPHICS.search(stripped)
|
|
if m:
|
|
current_gpu.clock_graphics = f"{m.group(1)} MHz"
|
|
continue
|
|
|
|
# HW Slowdown (순서 중요: 더 구체적인 패턴 먼저)
|
|
m = RE_HW_THERMAL_SLOWDOWN.search(stripped)
|
|
if m:
|
|
current_gpu.hw_thermal_slowdown = m.group(1)
|
|
continue
|
|
m = RE_HW_POWER_BRAKE_SLOWDOWN.search(stripped)
|
|
if m:
|
|
current_gpu.hw_power_brake_slowdown = m.group(1)
|
|
continue
|
|
m = RE_HW_SLOWDOWN.search(stripped)
|
|
if m:
|
|
current_gpu.hw_slowdown = m.group(1)
|
|
continue
|
|
|
|
# --- RMA: ECC 섹션 파싱 (GPU 내부) ---
|
|
if RE_ECC_SECTION.match(stripped):
|
|
in_ecc_section = True
|
|
ecc_subsection = None
|
|
continue
|
|
if in_ecc_section:
|
|
if RE_VOLATILE.match(stripped):
|
|
ecc_subsection = "volatile"
|
|
continue
|
|
if RE_AGGREGATE_SECTION.match(stripped):
|
|
ecc_subsection = "aggregate"
|
|
continue
|
|
if RE_AGG_SRAM_SOURCES.search(stripped):
|
|
ecc_subsection = "agg_sources"
|
|
continue
|
|
|
|
if ecc_subsection == "volatile":
|
|
m = RE_SRAM_CORR.search(stripped)
|
|
if m: current_gpu.sram_corr_volatile = m.group(1); continue
|
|
m = RE_SRAM_UNCORR_PARITY.search(stripped)
|
|
if m: current_gpu.sram_uncorr_parity_volatile = m.group(1); continue
|
|
m = RE_SRAM_UNCORR_SECDED.search(stripped)
|
|
if m: current_gpu.sram_uncorr_secded_volatile = m.group(1); continue
|
|
m = RE_DRAM_CORR.search(stripped)
|
|
if m: current_gpu.dram_corr_volatile = m.group(1); continue
|
|
m = RE_DRAM_UNCORR.search(stripped)
|
|
if m: current_gpu.dram_uncorr_volatile = m.group(1); continue
|
|
|
|
elif ecc_subsection == "aggregate":
|
|
m = RE_SRAM_CORR.search(stripped)
|
|
if m: current_gpu.sram_corr_aggregate = m.group(1); continue
|
|
m = RE_SRAM_UNCORR_PARITY.search(stripped)
|
|
if m: current_gpu.sram_uncorr_parity_aggregate = m.group(1); continue
|
|
m = RE_SRAM_UNCORR_SECDED.search(stripped)
|
|
if m: current_gpu.sram_uncorr_secded_aggregate = m.group(1); continue
|
|
m = RE_DRAM_CORR.search(stripped)
|
|
if m: current_gpu.dram_corr_aggregate = m.group(1); continue
|
|
m = RE_DRAM_UNCORR.search(stripped)
|
|
if m: current_gpu.dram_uncorr_aggregate = m.group(1); continue
|
|
m = RE_SRAM_THRESHOLD.search(stripped)
|
|
if m: current_gpu.sram_threshold_exceeded = m.group(1); continue
|
|
|
|
elif ecc_subsection == "agg_sources":
|
|
m = RE_SRAM_L2.search(stripped)
|
|
if m: current_gpu.sram_l2 = m.group(1); continue
|
|
m = RE_SRAM_SM.search(stripped)
|
|
if m: current_gpu.sram_sm = m.group(1); continue
|
|
m = RE_SRAM_MC.search(stripped)
|
|
if m: current_gpu.sram_microcontroller = m.group(1); continue
|
|
m = RE_SRAM_PCIE.search(stripped)
|
|
if m: current_gpu.sram_pcie = m.group(1); continue
|
|
m = RE_SRAM_OTHER.search(stripped)
|
|
if m: current_gpu.sram_other = m.group(1); continue
|
|
|
|
# --- RMA: Remapped Rows 섹션 파싱 (GPU 내부) ---
|
|
if RE_REMAP_SECTION.match(stripped):
|
|
in_remap_section = True
|
|
in_ecc_section = False
|
|
in_bank_histogram = False
|
|
continue
|
|
if in_remap_section:
|
|
m = RE_REMAP_CORR.search(stripped)
|
|
if m and "remap" not in stripped.lower():
|
|
current_gpu.remap_corr_error = int(m.group(1)); continue
|
|
m = RE_REMAP_UNCORR.search(stripped)
|
|
if m:
|
|
current_gpu.remap_uncorr_error = int(m.group(1)); continue
|
|
m = RE_REMAP_FAILURE.search(stripped)
|
|
if m:
|
|
current_gpu.remap_failure = m.group(1); continue
|
|
m = RE_REMAP_PENDING.search(stripped)
|
|
if m and "page" not in stripped.lower():
|
|
current_gpu.remap_pending = m.group(1); continue
|
|
if "Bank Remap Availability" in stripped:
|
|
in_bank_histogram = True; continue
|
|
if in_bank_histogram:
|
|
m = RE_BANK_MAX.search(stripped)
|
|
if m: current_gpu.remap_bank_max = int(m.group(1)); continue
|
|
m = RE_BANK_HIGH.search(stripped)
|
|
if m: current_gpu.remap_bank_high = int(m.group(1)); continue
|
|
m = RE_BANK_PARTIAL.search(stripped)
|
|
if m: current_gpu.remap_bank_partial = int(m.group(1)); continue
|
|
m = RE_BANK_LOW.search(stripped)
|
|
if m: current_gpu.remap_bank_low = int(m.group(1)); continue
|
|
m = RE_BANK_NONE.search(stripped)
|
|
if m:
|
|
current_gpu.remap_bank_none = int(m.group(1))
|
|
in_remap_section = False
|
|
in_bank_histogram = False
|
|
continue
|
|
|
|
# --- ECC 에러 전체 집계 (라인 단위 상태 머신) ---
|
|
if RE_SINGLE_BIT.search(stripped) and "sram" not in stripped.lower():
|
|
ecc_context = "single"
|
|
continue
|
|
if RE_DOUBLE_BIT.search(stripped) and "sram" not in stripped.lower():
|
|
ecc_context = "double"
|
|
continue
|
|
if ecc_context:
|
|
agg_m = RE_AGGREGATE.search(stripped)
|
|
if agg_m:
|
|
val = int(agg_m.group(1))
|
|
if ecc_context == "single":
|
|
ecc_single_total = max(ecc_single_total, val)
|
|
else:
|
|
ecc_double_total = max(ecc_double_total, val)
|
|
ecc_context = None
|
|
continue
|
|
|
|
ret_m = RE_RETIRED.search(stripped)
|
|
if ret_m:
|
|
ecc_retired = max(ecc_retired, int(ret_m.group(1)))
|
|
continue
|
|
if RE_PENDING.search(stripped) and "remap" not in stripped.lower():
|
|
ecc_pending = True
|
|
continue
|
|
|
|
# --- 커널 이벤트 (dmesg 섹션) ---
|
|
if in_dmesg and stripped not in seen_kernel:
|
|
if KERNEL_SKIP_KW.search(stripped):
|
|
continue
|
|
|
|
level = None
|
|
keyword = ""
|
|
# 우선순위: oom > critical > pcie > error
|
|
m = KERNEL_OOM_KW.search(stripped)
|
|
if m:
|
|
level, keyword = "oom", m.group(1)
|
|
if not level:
|
|
m = KERNEL_CRITICAL_KW.search(stripped)
|
|
if m:
|
|
level, keyword = "critical", m.group(1)
|
|
if not level:
|
|
m = KERNEL_PCIE_KW.search(stripped)
|
|
if m:
|
|
level, keyword = "pcie_error", m.group(1)
|
|
if not level:
|
|
m = KERNEL_ERROR_KW.search(stripped)
|
|
if m:
|
|
level, keyword = "error", m.group(1)
|
|
|
|
if level:
|
|
seen_kernel.add(stripped)
|
|
ts_m = RE_TIMESTAMP.match(stripped)
|
|
kernel_events.append(KernelEvent(
|
|
timestamp=ts_m.group(1) if ts_m else "",
|
|
level=level,
|
|
message=keyword,
|
|
raw_line=stripped[:500],
|
|
))
|
|
|
|
# --- 마지막 GPU 저장 ---
|
|
if current_gpu and current_gpu.name != "N/A" and not smi_gpus_found:
|
|
gpu_list.append(current_gpu)
|
|
|
|
# --- 결과 조립 ---
|
|
result.driver_version = driver_version or "N/A"
|
|
result.cuda_version = cuda_version or "N/A"
|
|
|
|
for gpu in gpu_list:
|
|
gpu.driver_version = result.driver_version
|
|
gpu.cuda_version = result.cuda_version
|
|
result.gpus = gpu_list
|
|
|
|
# gpu_list 시리얼 보완: NVSMI LOG에서 수집한 pci_serial_map 활용
|
|
for gpu in gpu_list:
|
|
if (not gpu.serial_number or gpu.serial_number == "N/A") and gpu.pci_address:
|
|
s = pci_serial_map.get(_norm_pci(gpu.pci_address))
|
|
if s:
|
|
gpu.serial_number = s
|
|
|
|
# PCI 주소 → GPU 매핑 (정규화 키 사용)
|
|
pci_to_gpu = {}
|
|
for gpu in gpu_list:
|
|
if gpu.pci_address and gpu.pci_address != "N/A":
|
|
pci_to_gpu[_norm_pci(gpu.pci_address)] = gpu
|
|
for event in xid_events:
|
|
key = _norm_pci(event.pci_address)
|
|
matched = pci_to_gpu.get(key)
|
|
if matched:
|
|
event.gpu_index = matched.index
|
|
event.gpu_name = matched.name
|
|
event.gpu_serial = matched.serial_number or pci_serial_map.get(key, "")
|
|
else:
|
|
# gpu_list에 없어도 pci_serial_map에서 시리얼 보완
|
|
event.gpu_serial = pci_serial_map.get(key, "")
|
|
|
|
result.xid_events = xid_events
|
|
xid_counter = Counter(e.xid_code for e in xid_events)
|
|
result.xid_summary = {
|
|
code: {
|
|
"count": count,
|
|
"severity": XID_DB["xid_errors"].get(str(code), {}).get("severity", "warning"),
|
|
"name": XID_DB["xid_errors"].get(str(code), {}).get("name", f"Unknown XID {code}"),
|
|
"gpus": sorted({
|
|
f"GPU {e.gpu_index} {e.gpu_name} [S/N: {e.gpu_serial}]" if e.gpu_serial
|
|
else (f"GPU {e.gpu_index} ({e.gpu_name})" if e.gpu_name else e.pci_address)
|
|
for e in xid_events if e.xid_code == code
|
|
}),
|
|
}
|
|
for code, count in xid_counter.most_common()
|
|
}
|
|
|
|
result.kernel_events = kernel_events
|
|
kernel_counter = Counter(e.level for e in kernel_events)
|
|
result.kernel_summary = dict(kernel_counter)
|
|
|
|
result.ecc_info = {
|
|
"single_bit_total": ecc_single_total,
|
|
"double_bit_total": ecc_double_total,
|
|
"retired_pages": ecc_retired,
|
|
"pending_retirement": ecc_pending,
|
|
}
|
|
|
|
# --- RMA 판정 ---
|
|
result.rma_verdicts = [evaluate_rma(gpu) for gpu in gpu_list]
|
|
result.rma_count = sum(1 for v in result.rma_verdicts if v.qualifies_for_rma)
|
|
|
|
# 상태 판정
|
|
critical_xids = sum(1 for e in xid_events if e.severity == "critical")
|
|
warning_xids = sum(1 for e in xid_events if e.severity == "warning")
|
|
|
|
result.critical_count = (
|
|
critical_xids
|
|
+ kernel_counter.get("critical", 0)
|
|
+ kernel_counter.get("oom", 0)
|
|
+ (1 if ecc_double_total > 0 else 0)
|
|
+ result.rma_count # RMA 대상 GPU 수도 critical에 포함
|
|
)
|
|
result.warning_count = (
|
|
warning_xids
|
|
+ kernel_counter.get("error", 0)
|
|
+ kernel_counter.get("pcie_error", 0)
|
|
+ (1 if ecc_single_total > 10 else 0)
|
|
)
|
|
result.info_count = sum(1 for e in xid_events if e.severity == "info")
|
|
result.total_issues = result.critical_count + result.warning_count
|
|
|
|
if result.rma_count > 0:
|
|
result.overall_status = "critical"
|
|
elif result.critical_count > 0:
|
|
result.overall_status = "critical"
|
|
elif result.warning_count > 0:
|
|
result.overall_status = "warning"
|
|
else:
|
|
result.overall_status = "healthy"
|
|
|
|
return result
|
|
|
|
|
|
def _safe_int(val: str) -> int:
|
|
"""N/A 등 숫자가 아닌 값을 0으로 변환"""
|
|
try:
|
|
return int(val)
|
|
except (ValueError, TypeError):
|
|
return 0
|
|
|
|
|
|
def evaluate_rma(gpu: GpuInfo) -> RmaVerdict:
|
|
"""
|
|
NVIDIA Hopper RMA 기준에 따라 GPU의 RMA 적격 여부를 판정합니다.
|
|
|
|
[Row-Remapping RMA 기준]
|
|
1. Remapping Failure Occurred = Yes → 즉시 RMA
|
|
2. 한 Bank에서 Uncorrectable Error Row 8회 이상 리매핑
|
|
3. 총 512회 리매핑 발생
|
|
4. 리매핑 8회 후 Field Diagnostics에서 문제 감지
|
|
|
|
[SRAM RMA 기준]
|
|
1. SRAM Threshold Exceeded = Yes → 즉시 RMA
|
|
2. Parity SRAM: 주소 뱅크 내 4개 이상 UCE Unique Count
|
|
3. SECDED ECC SRAM: 주소 뱅크 내 2개 이상 UCE Unique Count
|
|
"""
|
|
verdict = RmaVerdict(
|
|
gpu_index=gpu.index,
|
|
gpu_name=gpu.name,
|
|
serial_number=gpu.serial_number,
|
|
)
|
|
|
|
# --- Row-Remapping 분석 ---
|
|
total_remaps = gpu.remap_corr_error + gpu.remap_uncorr_error
|
|
|
|
verdict.row_remap_details = {
|
|
"correctable_error": gpu.remap_corr_error,
|
|
"uncorrectable_error": gpu.remap_uncorr_error,
|
|
"total_remaps": total_remaps,
|
|
"pending": gpu.remap_pending,
|
|
"failure_occurred": gpu.remap_failure,
|
|
"bank_max": gpu.remap_bank_max,
|
|
"bank_high": gpu.remap_bank_high,
|
|
"bank_partial": gpu.remap_bank_partial,
|
|
"bank_low": gpu.remap_bank_low,
|
|
"bank_none": gpu.remap_bank_none,
|
|
}
|
|
|
|
# RMA 기준 1: Remapping Failure Occurred = Yes
|
|
if gpu.remap_failure.lower() == "yes":
|
|
verdict.qualifies_for_rma = True
|
|
verdict.reasons.append(
|
|
"Remapping Failure Occurred = Yes: "
|
|
"리매핑 실패 플래그 설정됨. 즉시 RMA 대상."
|
|
)
|
|
|
|
# RMA 기준 2: Uncorrectable Error Row 8회 이상 (Bank당)
|
|
if gpu.remap_uncorr_error >= 8:
|
|
verdict.qualifies_for_rma = True
|
|
verdict.reasons.append(
|
|
f"Uncorrectable Error 리매핑 {gpu.remap_uncorr_error}회: "
|
|
f"Bank당 8회 이상 Uncorrectable Error 리매핑 발생 (기준: 8회)."
|
|
)
|
|
|
|
# RMA 기준 3: 총 512회 리매핑
|
|
if total_remaps >= 512:
|
|
verdict.qualifies_for_rma = True
|
|
verdict.reasons.append(
|
|
f"총 리매핑 {total_remaps}회: "
|
|
f"Uncorrectable Memory Error로 총 512회 이상 리매핑 발생 (기준: 512회)."
|
|
)
|
|
|
|
# 경고: 리매핑 진행 중
|
|
if gpu.remap_pending.lower() == "yes":
|
|
verdict.warnings.append(
|
|
"Row Remapping Pending = Yes: 리매핑 대기 중. 재부팅 후 적용됩니다."
|
|
)
|
|
|
|
# 경고: Bank 가용성 낮음
|
|
if gpu.remap_bank_none > 0:
|
|
verdict.warnings.append(
|
|
f"Bank Remap None = {gpu.remap_bank_none}: "
|
|
f"리매핑 가용 뱅크가 소진된 뱅크가 있습니다."
|
|
)
|
|
if gpu.remap_bank_low > 0:
|
|
verdict.warnings.append(
|
|
f"Bank Remap Low = {gpu.remap_bank_low}: "
|
|
f"리매핑 가용 뱅크가 적은 뱅크가 있습니다."
|
|
)
|
|
|
|
# 경고: 리매핑이 상당수 발생
|
|
if 0 < gpu.remap_uncorr_error < 8:
|
|
verdict.warnings.append(
|
|
f"Uncorrectable Error 리매핑 {gpu.remap_uncorr_error}회: "
|
|
f"아직 RMA 기준(8회) 미달이나 모니터링 필요."
|
|
)
|
|
|
|
# --- SRAM 분석 ---
|
|
sram_uncorr_parity = _safe_int(gpu.sram_uncorr_parity_aggregate)
|
|
sram_uncorr_secded = _safe_int(gpu.sram_uncorr_secded_aggregate)
|
|
|
|
verdict.sram_details = {
|
|
"volatile": {
|
|
"correctable": gpu.sram_corr_volatile,
|
|
"uncorrectable_parity": gpu.sram_uncorr_parity_volatile,
|
|
"uncorrectable_secded": gpu.sram_uncorr_secded_volatile,
|
|
},
|
|
"aggregate": {
|
|
"correctable": gpu.sram_corr_aggregate,
|
|
"uncorrectable_parity": gpu.sram_uncorr_parity_aggregate,
|
|
"uncorrectable_secded": gpu.sram_uncorr_secded_aggregate,
|
|
"threshold_exceeded": gpu.sram_threshold_exceeded,
|
|
},
|
|
"sources": {
|
|
"l2": gpu.sram_l2,
|
|
"sm": gpu.sram_sm,
|
|
"microcontroller": gpu.sram_microcontroller,
|
|
"pcie": gpu.sram_pcie,
|
|
"other": gpu.sram_other,
|
|
},
|
|
"dram": {
|
|
"corr_volatile": gpu.dram_corr_volatile,
|
|
"uncorr_volatile": gpu.dram_uncorr_volatile,
|
|
"corr_aggregate": gpu.dram_corr_aggregate,
|
|
"uncorr_aggregate": gpu.dram_uncorr_aggregate,
|
|
},
|
|
}
|
|
|
|
# SRAM RMA 기준 1: Threshold Exceeded = Yes
|
|
if gpu.sram_threshold_exceeded.lower() == "yes":
|
|
verdict.qualifies_for_rma = True
|
|
verdict.reasons.append(
|
|
"SRAM Threshold Exceeded = Yes: "
|
|
"SRAM UCE 임계치 초과 플래그 설정됨. 즉시 RMA 대상."
|
|
)
|
|
|
|
# SRAM RMA 기준 2: Parity SRAM UCE >= 4
|
|
if sram_uncorr_parity >= 4:
|
|
verdict.qualifies_for_rma = True
|
|
verdict.reasons.append(
|
|
f"SRAM Uncorrectable Parity (Aggregate) = {sram_uncorr_parity}: "
|
|
f"Parity 보호 SRAM에서 4개 이상 UCE 발생 (기준: 4개)."
|
|
)
|
|
|
|
# SRAM RMA 기준 3: SECDED ECC SRAM UCE >= 2
|
|
if sram_uncorr_secded >= 2:
|
|
verdict.qualifies_for_rma = True
|
|
verdict.reasons.append(
|
|
f"SRAM Uncorrectable SEC-DED (Aggregate) = {sram_uncorr_secded}: "
|
|
f"SECDED ECC 보호 SRAM에서 2개 이상 UCE 발생 (기준: 2개)."
|
|
)
|
|
|
|
# 경고: SRAM UCE가 있지만 기준 미달
|
|
if 0 < sram_uncorr_parity < 4:
|
|
verdict.warnings.append(
|
|
f"SRAM Parity UCE = {sram_uncorr_parity}: "
|
|
f"아직 RMA 기준(4개) 미달이나 모니터링 필요."
|
|
)
|
|
if 0 < sram_uncorr_secded < 2:
|
|
verdict.warnings.append(
|
|
f"SRAM SECDED UCE = {sram_uncorr_secded}: "
|
|
f"아직 RMA 기준(2개) 미달이나 모니터링 필요."
|
|
)
|
|
|
|
return verdict
|