commit 59e3a62ee5b8cc5b0e7129fd975dbdd26b708423 Author: unknown Date: Wed Jun 24 14:43:05 2026 +0900 feat: merge Xid-Catalog.xlsx, fix dmesg regex, and improve history UI diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d1c8ff0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +__pycache__ +*.pyc +.git +.gitignore +*.md +.env diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dc9b0d4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,31 @@ +# Python +__pycache__/ +*.pyc +*.pyo +*.pyd +.Python +env/ +venv/ +.venv/ +pip-log.txt +pip-delete-this-directory.txt + +# Databases +data/nv_log_history.db + +# Backup files +*.bak + +# IDEs and editors +.vscode/ +.idea/ +.claude/ +*.suo +*.ntvs* +*.njsproj +*.sln +*.swp + +# OS generated files +.DS_Store +Thumbs.db diff --git a/.streamlit/config.toml b/.streamlit/config.toml new file mode 100644 index 0000000..f8ec6a8 --- /dev/null +++ b/.streamlit/config.toml @@ -0,0 +1,2 @@ +[client] +toolbarMode = "minimal" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..917aea4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.11-slim + +WORKDIR /app + +# 시스템 의존성 설치 +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +# 파이썬 패키지 설치 +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# 애플리케이션 복사 +COPY . . + +# Streamlit 포트 +EXPOSE 8501 + +# 헬스체크 +HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health || exit 1 + +# Streamlit 실행 +ENTRYPOINT ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0", "--server.headless=true"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..00553d6 --- /dev/null +++ b/README.md @@ -0,0 +1,79 @@ +# NV-Log Analyzer + +NVIDIA Bug Report 로그 자가 진단 서비스 + +`nvidia-bug-report.log.gz` 파일을 업로드하면 XID 에러, GPU 상태, 커널 로그를 자동 분석하여 권장 조치를 안내합니다. + +## 주요 기능 + +- **XID 에러 탐지**: `NVRM: Xid` 패턴 자동 검출 + 에러 코드별 한글 권장 조치 제공 +- **GPU 헬스 체크**: 온도, 전력, ECC 에러, 메모리 상태 요약 +- **커널 로그 필터링**: dmesg 내 Critical, Error, OOM, PCIe 에러 추출 +- **원본 로그 검색**: 키워드 기반 로그 검색 기능 +- **XID 코드 조회**: 사이드바에서 XID 번호로 즉시 의미 확인 + +## 빠른 시작 + +### 방법 1: 로컬 실행 + +```bash +pip install -r requirements.txt +streamlit run app.py +``` + +브라우저에서 `http://localhost:8501` 접속 + +### 방법 2: Docker + +```bash +docker-compose up -d +``` + +브라우저에서 `http://localhost:8501` 접속 + +## 프로젝트 구조 + +``` +nvidia/ +├── app.py # Streamlit 대시보드 (메인 UI) +├── log_parser.py # 핵심 분석 엔진 +├── data/ +│ └── xid_database.json # XID 에러 코드 매핑 DB (18종) +├── requirements.txt +├── Dockerfile +├── docker-compose.yml +└── README.md +``` + +## 분석 대상 로그 생성 방법 + +GPU 서버에서 아래 명령 실행: + +```bash +sudo nvidia-bug-report.sh +``` + +생성된 `nvidia-bug-report.log.gz` 파일을 웹 대시보드에 업로드하면 됩니다. + +## 지원하는 XID 코드 + +| XID | 이름 | 심각도 | +|-----|------|--------| +| 8 | GPU Stopped Responding | 심각 | +| 13 | Graphics Engine Exception | 심각 | +| 31 | GPU Memory Page Fault | 심각 | +| 32 | Invalid Push Buffer | 경고 | +| 38 | Driver Firmware Mismatch | 경고 | +| 43 | GPU Stopped Processing | 심각 | +| 45 | GPU Preemption Failure | 경고 | +| 48 | Double Bit ECC Error | 심각 | +| 61 | GPU Fallen Off the Bus | 심각 | +| 62 | GPU ECC Page Retirement | 경고 | +| 63 | ECC Page Retirement: Double Bit | 심각 | +| 64 | ECC Page Retirement Failure | 심각 | +| 69 | GPU Access to System Memory Failed | 경고 | +| 79 | GPU Fallen Off the Bus (Fatal) | 심각 | +| 92 | High Single-bit ECC Error Rate | 경고 | +| 94 | Contained ECC Error | 정보 | +| 95 | Uncontained ECC Error | 심각 | +| 119 | GPU Recovery Action | 정보 | diff --git a/Xid-Catalog.xlsx b/Xid-Catalog.xlsx new file mode 100644 index 0000000..320d869 Binary files /dev/null and b/Xid-Catalog.xlsx differ diff --git a/app.py b/app.py new file mode 100644 index 0000000..967d300 --- /dev/null +++ b/app.py @@ -0,0 +1,1087 @@ +""" +NV-Log Analyzer - Streamlit 대시보드 +nvidia-bug-report.log.gz 파일을 업로드하면 즉시 분석 + DB 저장. +이력 조회, GPU 시리얼 추적, XID 통계 등 케이스 축적 기능 포함. +""" + +import streamlit as st +import pandas as pd +import json +from pathlib import Path +from log_parser import analyze_log, read_log_file, XID_DB +from db import init_db, save_analysis, update_memo, delete_analysis +from db import get_analysis_list, get_analysis_detail +from db import get_xid_statistics, get_gpu_serial_history, get_all_gpu_serials, get_status_summary +from auth import ( + init_user_db, authenticate, register_user, get_user_permissions, + get_all_users, get_pending_users, approve_user, reject_user, + update_user_role, delete_user, update_password, ROLES, +) + +# --- DB 초기화 (앱 시작 시 1회) --- +init_db() +init_user_db() + +# --- 페이지 설정 --- +st.set_page_config( + page_title="NV-Log Analyzer", + page_icon="🖥️", + layout="wide", + initial_sidebar_state="expanded", +) + +# --- 커스텀 CSS --- +st.markdown(""" + +""", unsafe_allow_html=True) + + +# ============================================= +# 인증 헬퍼 +# ============================================= + +def get_current_user(): + """현재 로그인된 사용자 정보 반환. 없으면 None""" + return st.session_state.get("user") + + +def get_perms(): + """현재 사용자 권한 dict""" + user = get_current_user() + if not user: + return ROLES["viewer"] + return get_user_permissions(user["role"]) + + +def require_login(): + """로그인 안 되었으면 로그인 페이지 표시 후 True 반환 (= 중단 필요)""" + if get_current_user(): + return False + page_login() + return True + + +# ============================================= +# 로그인 / 회원가입 페이지 +# ============================================= + +def page_login(): + # 고정 너비 중앙 정렬 + col_left, col_center, col_right = st.columns([1, 2, 1]) + + with col_center: + st.markdown('

🖥️ NV-Log Analyzer

', + unsafe_allow_html=True) + + tab_login, tab_register = st.tabs(["🔑 로그인", "📝 회원가입"]) + + with tab_login: + with st.form("login_form"): + username = st.text_input("아이디") + password = st.text_input("비밀번호", type="password") + submitted = st.form_submit_button("로그인", use_container_width=True) + + if submitted and username and password: + result = authenticate(username, password) + if result is None: + st.error("아이디 또는 비밀번호가 올바르지 않습니다.") + elif "error" in result: + if result["status"] == "pending": + st.warning("가입 승인 대기 중입니다. 관리자 승인 후 로그인할 수 있습니다.") + elif result["status"] == "rejected": + st.error("가입이 거절되었습니다. 관리자에게 문의하세요.") + else: + st.session_state["user"] = result + st.rerun() + + with tab_register: + with st.form("register_form"): + new_username = st.text_input("아이디 (3자 이상)") + new_display = st.text_input("표시 이름") + new_password = st.text_input("비밀번호 (6자 이상)", type="password") + new_password2 = st.text_input("비밀번호 확인", type="password") + reg_submitted = st.form_submit_button("회원가입 신청", use_container_width=True) + + if reg_submitted: + if new_password != new_password2: + st.error("비밀번호가 일치하지 않습니다.") + elif new_username and new_password: + res = register_user(new_username, new_password, new_display or new_username) + if res["success"]: + st.success("회원가입이 신청되었습니다. 관리자 승인 후 로그인할 수 있습니다.") + else: + st.error(res["error"]) + + st.markdown("---") + st.caption("기본 관리자: admin / admin1234 (최초 로그인 후 반드시 비밀번호를 변경하세요)") + + +# ============================================= +# 페이지 5: 사용자 관리 (admin 전용) +# ============================================= + +def page_user_management(): + st.markdown('

👤 사용자 관리

', unsafe_allow_html=True) + st.markdown('

사용자 가입 승인, 역할 변경, 계정 관리를 수행합니다.

', + unsafe_allow_html=True) + + user = get_current_user() + if not user or user["role"] != "admin": + st.error("관리자만 접근할 수 있습니다.") + return + + # 승인 대기 사용자 + pending = get_pending_users() + if pending: + st.subheader(f"🔔 승인 대기 ({len(pending)}명)") + for p in pending: + with st.container(): + pc1, pc2, pc3, pc4 = st.columns([2, 2, 2, 2]) + pc1.write(f"**{p['display_name']}** (`{p['username']}`)") + pc2.write(f"가입 요청: {p['created_at']}") + with pc3: + role_choice = st.selectbox( + "역할", list(ROLES.keys()), + index=2, # default viewer + key=f"role_{p['id']}", + format_func=lambda r: f"{ROLES[r]['label']} - {ROLES[r]['description'][:20]}", + ) + with pc4: + bc1, bc2 = st.columns(2) + if bc1.button("✅ 승인", key=f"approve_{p['id']}"): + approve_user(p["id"], role_choice, user["username"]) + st.success(f"{p['username']} 승인 완료 ({ROLES[role_choice]['label']})") + st.rerun() + if bc2.button("❌ 거절", key=f"reject_{p['id']}"): + reject_user(p["id"]) + st.warning(f"{p['username']} 거절됨") + st.rerun() + st.markdown("---") + else: + st.info("승인 대기 중인 사용자가 없습니다.") + + # 전체 사용자 목록 + st.subheader("📋 전체 사용자") + all_users = get_all_users() + + status_emoji = {"approved": "🟢", "pending": "🟡", "rejected": "🔴"} + user_table = [{ + "ID": u["id"], + "아이디": u["username"], + "이름": u["display_name"], + "역할": f"{ROLES.get(u['role'], {}).get('label', u['role'])}", + "상태": f"{status_emoji.get(u['status'], '')} {u['status']}", + "가입일": u["created_at"], + "승인자": u.get("approved_by") or "-", + "최근 로그인": u.get("last_login") or "-", + } for u in all_users] + st.dataframe(pd.DataFrame(user_table), use_container_width=True, hide_index=True) + + # 개별 사용자 관리 + st.markdown("---") + st.subheader("🔧 사용자 설정 변경") + + manageable = [u for u in all_users if u["username"] != "admin"] + if not manageable: + st.info("관리할 사용자가 없습니다.") + return + + selected_user = st.selectbox( + "사용자 선택", + manageable, + format_func=lambda u: f"{u['display_name']} ({u['username']}) - {ROLES.get(u['role'], {}).get('label', u['role'])}", + ) + + if selected_user: + mc1, mc2, mc3 = st.columns(3) + + with mc1: + new_role = st.selectbox( + "역할 변경", + list(ROLES.keys()), + index=list(ROLES.keys()).index(selected_user["role"]), + key="change_role", + format_func=lambda r: f"{ROLES[r]['label']}", + ) + if st.button("역할 적용"): + update_user_role(selected_user["id"], new_role) + st.success(f"{selected_user['username']}의 역할이 {ROLES[new_role]['label']}(으)로 변경되었습니다.") + st.rerun() + + with mc2: + new_pw = st.text_input("새 비밀번호 (6자 이상)", type="password", key="reset_pw") + if st.button("비밀번호 초기화"): + if new_pw and len(new_pw) >= 6: + update_password(selected_user["id"], new_pw) + st.success(f"{selected_user['username']}의 비밀번호가 변경되었습니다.") + else: + st.error("비밀번호는 6자 이상이어야 합니다.") + + with mc3: + st.write("") + st.write("") + if st.button("🗑️ 계정 삭제", type="secondary"): + delete_user(selected_user["id"]) + st.success(f"{selected_user['username']} 계정이 삭제되었습니다.") + st.rerun() + + # 역할별 권한 안내 + st.markdown("---") + with st.expander("📖 역할별 권한 안내"): + for role_key, role_info in ROLES.items(): + perms = [] + if role_info["can_upload"]: perms.append("로그 업로드") + if role_info["can_view"]: perms.append("조회") + if role_info["can_edit_memo"]: perms.append("메모 수정") + if role_info["can_delete"]: perms.append("삭제") + if role_info["can_manage_users"]: perms.append("사용자 관리") + st.write(f"**{role_info['label']}** ({role_key}): {', '.join(perms)}") + + +# ============================================= +# 공통 렌더링 함수 +# ============================================= + +def render_status_badge(status): + status_map = { + "critical": ("🔴 심각", "status-critical", + "즉각적인 조치가 필요합니다. 하드웨어 손상 또는 서비스 중단 위험이 감지되었습니다."), + "warning": ("🟡 경고", "status-warning", + "주의가 필요한 항목이 발견되었습니다. 방치 시 문제가 악화될 수 있습니다."), + "healthy": ("🟢 정상", "status-healthy", + "심각한 문제가 감지되지 않았습니다."), + } + label, css_class, desc = status_map.get(status, status_map["healthy"]) + st.markdown(f'

{label}

' + f'

{desc}

', unsafe_allow_html=True) + + +def render_gpu_details_from_result(gpus): + """GPU 상세 (AnalysisResult.gpus 객체 리스트용)""" + if not gpus: + st.info("GPU 정보를 로그에서 추출하지 못했습니다.") + return + for gpu in gpus: + with st.expander(f"GPU {gpu.index}: {gpu.name}", expanded=True): + col1, col2, col3, col4 = st.columns(4) + temp_delta = "정상" if gpu.temperature < 85 else "과열 주의!" + col1.metric("온도", f"{gpu.temperature}°C", delta=temp_delta, + delta_color="off" if gpu.temperature < 85 else "inverse") + if gpu.power_limit > 0: + col2.metric("전력", f"{gpu.power_draw}W / {gpu.power_limit}W", + delta=f"{round(gpu.power_draw/gpu.power_limit*100)}%") + else: + col2.metric("전력", f"{gpu.power_draw}W") + if gpu.memory_total > 0: + col3.metric("메모리", f"{gpu.memory_used} / {gpu.memory_total} MiB", + delta=f"{round(gpu.memory_used/gpu.memory_total*100)}%") + else: + col3.metric("메모리", "N/A") + col4.metric("GPU 사용률", f"{gpu.gpu_utilization}%") + + spec_col1, spec_col2 = st.columns(2) + with spec_col1: + for k, v in {"제품명": gpu.name, "브랜드": gpu.product_brand, + "아키텍처": gpu.product_architecture, + "시리얼 번호": gpu.serial_number, + "GPU UUID": gpu.gpu_uuid, + "VBIOS": gpu.vbios_version, + "InfoROM": gpu.inforom_version}.items(): + if v and v != "N/A": + st.write(f"**{k}:** `{v}`") + with spec_col2: + pcie = f"{gpu.pci_link_gen_current} x{gpu.pci_link_width_current}" if gpu.pci_link_gen_current != "N/A" else "N/A" + for k, v in {"PCI 주소": gpu.pci_address, "PCIe 링크": pcie, + "Power State": gpu.power_state, + "팬 속도": f"{gpu.fan_speed}%" if gpu.fan_speed > 0 else "N/A", + "Persistence": gpu.persistence_mode, + "Compute Mode": gpu.compute_mode, + "MIG Mode": gpu.mig_mode, + "ECC Mode": gpu.ecc_mode}.items(): + if v and v != "N/A": + st.write(f"**{k}:** `{v}`") + + if gpu.clock_graphics != "N/A" or gpu.clock_memory != "N/A": + st.markdown("**클럭 정보**") + cc1, cc2 = st.columns(2) + if gpu.clock_graphics != "N/A": cc1.write(f"**Graphics:** `{gpu.clock_graphics}`") + if gpu.clock_memory != "N/A": cc2.write(f"**Memory:** `{gpu.clock_memory}`") + + if gpu.hw_slowdown != "N/A": + hw_sd = gpu.hw_slowdown + hw_th = gpu.hw_thermal_slowdown + hw_pb = gpu.hw_power_brake_slowdown + any_active = hw_sd == "Active" or hw_th == "Active" or hw_pb == "Active" + label = "⚠️ **HW Slowdown 상태**" if any_active else "**HW Slowdown 상태**" + st.markdown(label) + sd1, sd2, sd3 = st.columns(3) + sd1.metric("HW Slowdown", hw_sd, + delta="Active" if hw_sd == "Active" else None, + delta_color="inverse") + if hw_th != "N/A": + sd2.metric("HW Thermal Slowdown", hw_th, + delta="Active" if hw_th == "Active" else None, + delta_color="inverse") + if hw_pb != "N/A": + sd3.metric("HW Power Brake Slowdown", hw_pb, + delta="Active" if hw_pb == "Active" else None, + delta_color="inverse") + + +def render_gpu_details_from_dict(gpus): + """GPU 상세 (DB dict 리스트용)""" + if not gpus: + st.info("GPU 정보 없음") + return + for gpu in gpus: + with st.expander(f"GPU {gpu['gpu_index']}: {gpu['name']}", expanded=True): + col1, col2, col3, col4 = st.columns(4) + temp = gpu.get("temperature", 0) or 0 + col1.metric("온도", f"{temp}°C", + delta="정상" if temp < 85 else "과열 주의!", + delta_color="off" if temp < 85 else "inverse") + pw = gpu.get("power_draw", 0) or 0 + pl = gpu.get("power_limit", 0) or 0 + if pl > 0: + col2.metric("전력", f"{pw}W / {pl}W", delta=f"{round(pw/pl*100)}%") + else: + col2.metric("전력", f"{pw}W") + mu = gpu.get("memory_used", 0) or 0 + mt = gpu.get("memory_total", 0) or 0 + if mt > 0: + col3.metric("메모리", f"{mu} / {mt} MiB", delta=f"{round(mu/mt*100)}%") + else: + col3.metric("메모리", "N/A") + col4.metric("GPU 사용률", f"{gpu.get('gpu_utilization', 0)}%") + + spec_col1, spec_col2 = st.columns(2) + with spec_col1: + for k, field in [("제품명", "name"), ("브랜드", "product_brand"), + ("아키텍처", "product_architecture"), + ("시리얼 번호", "serial_number"), + ("GPU UUID", "gpu_uuid"), + ("VBIOS", "vbios_version"), + ("InfoROM", "inforom_version")]: + v = gpu.get(field, "N/A") or "N/A" + if v != "N/A": + st.write(f"**{k}:** `{v}`") + with spec_col2: + gen = gpu.get("pci_link_gen", "N/A") or "N/A" + width = gpu.get("pci_link_width", "N/A") or "N/A" + pcie = f"{gen} x{width}" if gen != "N/A" else "N/A" + fan = gpu.get("fan_speed", 0) or 0 + for k, v in [("PCI 주소", gpu.get("pci_address", "N/A")), + ("PCIe 링크", pcie), + ("Power State", gpu.get("power_state", "N/A")), + ("팬 속도", f"{fan}%" if fan > 0 else "N/A"), + ("Persistence", gpu.get("persistence_mode", "N/A")), + ("Compute Mode", gpu.get("compute_mode", "N/A")), + ("MIG Mode", gpu.get("mig_mode", "N/A")), + ("ECC Mode", gpu.get("ecc_mode", "N/A"))]: + if v and v != "N/A": + st.write(f"**{k}:** `{v}`") + + hw_sd = gpu.get("hw_slowdown", "N/A") or "N/A" + if hw_sd != "N/A": + hw_th = gpu.get("hw_thermal_slowdown", "N/A") or "N/A" + hw_pb = gpu.get("hw_power_brake_slowdown", "N/A") or "N/A" + any_active = hw_sd == "Active" or hw_th == "Active" or hw_pb == "Active" + label = "⚠️ **HW Slowdown 상태**" if any_active else "**HW Slowdown 상태**" + st.markdown(label) + sd1, sd2, sd3 = st.columns(3) + sd1.metric("HW Slowdown", hw_sd, + delta="Active" if hw_sd == "Active" else None, + delta_color="inverse") + if hw_th != "N/A": + sd2.metric("HW Thermal Slowdown", hw_th, + delta="Active" if hw_th == "Active" else None, + delta_color="inverse") + if hw_pb != "N/A": + sd3.metric("HW Power Brake Slowdown", hw_pb, + delta="Active" if hw_pb == "Active" else None, + delta_color="inverse") + + +def render_rma_section(rma_verdicts, source="result"): + """RMA 판정 결과 섹션 (AnalysisResult.rma_verdicts 또는 DB dict 리스트 공용)""" + st.subheader("🔧 RMA 적격 판정 (NVIDIA Hopper 기준)") + + if not rma_verdicts: + st.info("RMA 판정 데이터가 없습니다.") + return + + rma_qualified = [] + rma_warnings = [] + rma_clean = [] + + for v in rma_verdicts: + if source == "result": + # AnalysisResult.rma_verdicts (RmaVerdict 객체) + qualifies = v.qualifies_for_rma + idx = v.gpu_index + name = v.gpu_name + serial = v.serial_number + reasons = v.reasons + warnings = v.warnings + remap = v.row_remap_details + sram = v.sram_details + else: + # DB dict + qualifies = v.get("qualifies_for_rma", 0) == 1 + idx = v.get("gpu_index", 0) + name = v.get("gpu_name", "N/A") + serial = v.get("serial_number", "N/A") + reasons = v.get("reasons", []) + warnings = v.get("warnings", []) + remap = v.get("row_remap_details", {}) + sram = v.get("sram_details", {}) + + entry = { + "idx": idx, "name": name, "serial": serial, + "qualifies": qualifies, "reasons": reasons, "warnings": warnings, + "remap": remap, "sram": sram, + } + if qualifies: + rma_qualified.append(entry) + elif warnings: + rma_warnings.append(entry) + else: + rma_clean.append(entry) + + # RMA 대상 GPU 요약 + if rma_qualified: + st.markdown(f""" +
+

🔴 RMA 대상 GPU: {len(rma_qualified)}개

+

아래 GPU는 NVIDIA RMA 기준에 해당합니다. 벤더사에 교체를 요청하세요.

+
+ """, unsafe_allow_html=True) + elif rma_warnings: + st.markdown(f""" +
+

🟡 주의 필요 GPU: {len(rma_warnings)}개

+

아직 RMA 기준 미달이나 모니터링이 필요한 GPU가 있습니다.

+
+ """, unsafe_allow_html=True) + else: + st.markdown(f""" +
+

🟢 RMA 해당 없음

+

모든 GPU가 RMA 기준 내 정상 범위입니다.

+
+ """, unsafe_allow_html=True) + + # GPU별 상세 + for entry in rma_qualified + rma_warnings + rma_clean: + status_icon = "🔴" if entry["qualifies"] else ("🟡" if entry["warnings"] else "🟢") + status_text = "RMA 대상" if entry["qualifies"] else ("주의 필요" if entry["warnings"] else "정상") + + with st.expander( + f"{status_icon} GPU {entry['idx']}: {entry['name']} (S/N: {entry['serial']}) — {status_text}", + expanded=entry["qualifies"] + ): + # RMA 사유 + if entry["reasons"]: + st.markdown("**🔴 RMA 사유:**") + for r in entry["reasons"]: + st.error(r) + + # 경고 + if entry["warnings"]: + st.markdown("**🟡 경고:**") + for w in entry["warnings"]: + st.warning(w) + + # Row Remapping 상세 + remap = entry["remap"] + if remap: + st.markdown("**Row Remapping 상태:**") + rc1, rc2, rc3, rc4 = st.columns(4) + rc1.metric("Correctable 리매핑", remap.get("correctable_error", 0)) + rc2.metric("Uncorrectable 리매핑", remap.get("uncorrectable_error", 0)) + rc3.metric("리매핑 대기", remap.get("pending", "N/A")) + rc4.metric("리매핑 실패", remap.get("failure_occurred", "N/A")) + + st.markdown("**Bank Remap 가용성:**") + bc1, bc2, bc3, bc4, bc5 = st.columns(5) + bc1.metric("Max", f"{remap.get('bank_max', 0)} banks") + bc2.metric("High", f"{remap.get('bank_high', 0)} banks") + bc3.metric("Partial", f"{remap.get('bank_partial', 0)} banks") + bc4.metric("Low", f"{remap.get('bank_low', 0)} banks") + bc5.metric("None", f"{remap.get('bank_none', 0)} banks") + + # SRAM 상세 + sram = entry["sram"] + if sram: + st.markdown("**SRAM ECC 상태:**") + agg = sram.get("aggregate", {}) + vol = sram.get("volatile", {}) + dram = sram.get("dram", {}) + + sc1, sc2, sc3 = st.columns(3) + with sc1: + st.markdown("*Aggregate (누적)*") + st.write(f"Correctable: `{agg.get('correctable', 'N/A')}`") + st.write(f"Parity UCE: `{agg.get('uncorrectable_parity', 'N/A')}`") + st.write(f"SECDED UCE: `{agg.get('uncorrectable_secded', 'N/A')}`") + threshold = agg.get("threshold_exceeded", "N/A") + if threshold.lower() == "yes": + st.write(f"**Threshold Exceeded: `{threshold}`** ← RMA 대상") + else: + st.write(f"Threshold Exceeded: `{threshold}`") + with sc2: + st.markdown("*Volatile (휘발성)*") + st.write(f"Correctable: `{vol.get('correctable', 'N/A')}`") + st.write(f"Parity UCE: `{vol.get('uncorrectable_parity', 'N/A')}`") + st.write(f"SECDED UCE: `{vol.get('uncorrectable_secded', 'N/A')}`") + with sc3: + st.markdown("*DRAM*") + st.write(f"Corr (Vol): `{dram.get('corr_volatile', 'N/A')}`") + st.write(f"Uncorr (Vol): `{dram.get('uncorr_volatile', 'N/A')}`") + st.write(f"Corr (Agg): `{dram.get('corr_aggregate', 'N/A')}`") + st.write(f"Uncorr (Agg): `{dram.get('uncorr_aggregate', 'N/A')}`") + + # SRAM Sources + sources = sram.get("sources", {}) + if any(v != "N/A" for v in sources.values()): + st.markdown("**Uncorrectable SRAM Sources:**") + ss1, ss2, ss3, ss4, ss5 = st.columns(5) + ss1.write(f"L2: `{sources.get('l2', 'N/A')}`") + ss2.write(f"SM: `{sources.get('sm', 'N/A')}`") + ss3.write(f"MC: `{sources.get('microcontroller', 'N/A')}`") + ss4.write(f"PCIe: `{sources.get('pcie', 'N/A')}`") + ss5.write(f"Other: `{sources.get('other', 'N/A')}`") + + if not entry["qualifies"] and not entry["warnings"]: + st.success("Row-Remapping, SRAM 모두 정상 범위입니다.") + + # RMA 기준 안내 + with st.expander("📖 NVIDIA Hopper RMA 기준 안내", expanded=False): + st.markdown(""" +**Row-Remapping RMA 기준 (DRAM)** +- `Remapping Failure Occurred = Yes` → 즉시 RMA +- 한 Bank에서 Uncorrectable Error Row 리매핑 **8회 이상** +- 총 리매핑 **512회** 이상 +- 8회 리매핑 후 Field Diagnostics에서 문제 감지 + +**SRAM RMA 기준 (L2 Cache)** +- `SRAM Threshold Exceeded = Yes` → 즉시 RMA +- Parity 보호 SRAM: 주소 뱅크 내 **UCE 4개** 이상 +- SECDED ECC 보호 SRAM: 주소 뱅크 내 **UCE 2개** 이상 + +> 참고: DRAM은 별도 RMA 기준 없음. NVIDIA Field Diagnostic 도구로 최종 확인 필요. +> [NVIDIA 공식 문서](https://docs.nvidia.com/deploy/a100-gpu-mem-error-mgmt/rma-policy-thresholds-for-row-remapping.html) + """) + + +# ============================================= +# 페이지 1: 로그 분석 (메인) +# ============================================= + +def page_analyze(): + st.markdown('

🖥️ NV-Log Analyzer

', unsafe_allow_html=True) + st.markdown('

nvidia-bug-report.log.gz 파일을 업로드하여 GPU 상태를 즉시 분석하고 DB에 저장합니다.

', + unsafe_allow_html=True) + + perms = get_perms() + if not perms["can_upload"]: + st.warning("로그 업로드 권한이 없습니다. 관리자에게 문의하세요.") + st.info("분석 이력 조회는 '📚 분석 이력' 메뉴에서 가능합니다.") + return + + uploaded_file = st.file_uploader( + "nvidia-bug-report.log.gz 또는 .log 파일 업로드", + type=["gz", "log", "txt"], + help="nvidia-bug-report.sh가 생성한 로그 파일을 업로드하세요.", + ) + + if uploaded_file is None: + st.markdown("---") + col1, col2, col3 = st.columns(3) + with col1: + st.markdown("#### 🔍 XID 에러 탐지\n`NVRM: Xid` 패턴을 자동 검출하고, 에러 코드별 의미와 권장 조치를 제공합니다.") + with col2: + st.markdown("#### 🌡️ GPU 헬스 체크\n온도, 전력, ECC 에러, 메모리 상태를 한눈에 확인할 수 있습니다.") + with col3: + st.markdown("#### 🐧 커널 로그 필터링\ndmesg 내 Critical, Error, OOM 등 핵심 이벤트만 추출합니다.") + st.info("위에서 nvidia-bug-report.log.gz 파일을 업로드하세요.") + return + + # 분석 실행 + with st.spinner("🔄 로그 분석 중..."): + file_content = uploaded_file.read() + result = analyze_log(file_content, uploaded_file.name) + + # DB 저장 (중복 방지: 같은 파일명+크기가 이미 있으면 스킵) + existing = get_analysis_list(limit=1, search=uploaded_file.name) + if existing and existing[0]["file_size_mb"] == result.file_size_mb: + analysis_id = existing[0]["id"] + st.info(f"이미 저장된 분석 결과입니다. (ID: {analysis_id})") + else: + analysis_id = save_analysis(result) + st.success(f"분석 결과가 DB에 저장되었습니다. (ID: {analysis_id})") + + # 메모 입력 (editor/admin만) + if perms["can_edit_memo"]: + memo = st.text_input("📝 이 분석에 대한 메모 (장애 상황, 조치 내용 등)", key="memo_input", + placeholder="예: Dell 서버 R750xa, GPU 4번 XID 61 반복 발생으로 RMA 접수") + if memo: + update_memo(analysis_id, memo) + st.caption("메모가 저장되었습니다.") + + # --- 결과 출력 --- + st.markdown("---") + render_status_badge(result.overall_status) + + col1, col2, col3, col4, col5, col6 = st.columns(6) + col1.metric("전체 이슈", result.total_issues) + col2.metric("심각", result.critical_count) + col3.metric("경고", result.warning_count) + col4.metric("정보", result.info_count) + col5.metric("GPU 수", len(result.gpus)) + col6.metric("🔧 RMA 대상", result.rma_count) + + st.markdown("---") + st.subheader("📋 시스템 정보") + ic1, ic2, ic3, ic4 = st.columns(4) + ic1.metric("파일명", result.filename[:30] + ("..." if len(result.filename) > 30 else "")) + ic2.metric("파일 크기", f"{result.file_size_mb} MB") + ic3.metric("드라이버", result.driver_version) + ic4.metric("CUDA", result.cuda_version) + + st.markdown("---") + st.subheader("🎮 GPU 상태") + render_gpu_details_from_result(result.gpus) + + # RMA 판정 + st.markdown("---") + render_rma_section(result.rma_verdicts) + + # XID 에러 + st.markdown("---") + st.subheader("⚠️ XID 에러 분석") + if not result.xid_events: + st.success("XID 에러가 감지되지 않았습니다.") + else: + st.write(f"**총 {len(result.xid_events)}건의 XID 이벤트 감지**") + summary_data = [] + for code, info in result.xid_summary.items(): + sev = XID_DB["severity_levels"].get(info["severity"], {}) + summary_data.append({ + "XID": code, "이름": info["name"], "횟수": info["count"], + "심각도": f"{sev.get('emoji','')} {sev.get('label', info['severity'])}", + "발생 GPU": ", ".join(info.get("gpus", [])) or "N/A", + }) + st.dataframe(pd.DataFrame(summary_data), use_container_width=True, hide_index=True) + + displayed = set() + for event in result.xid_events: + if event.xid_code in displayed: + continue + displayed.add(event.xid_code) + sev = XID_DB["severity_levels"].get(event.severity, {}) + gpu_label = f"GPU {event.gpu_index} ({event.gpu_name})" if event.gpu_name else event.pci_address + serial_label = event.gpu_serial if event.gpu_serial else "N/A" + with st.expander(f"{sev.get('emoji','ℹ️')} XID {event.xid_code}: {event.name} " + f"({result.xid_summary[event.xid_code]['count']}건)", + expanded=(event.severity == "critical")): + st.markdown(f"**발생 GPU:** {gpu_label}") + st.markdown(f"**시리얼 번호:** `{serial_label}`") + st.markdown(f"**PCI 주소:** `{event.pci_address}`") + st.markdown(f"**설명:** {event.description}") + st.markdown("**권장 조치:**") + for ln in event.action.split("\n"): + st.markdown(f" {ln}") + + # 커널 이벤트 + st.markdown("---") + st.subheader("🐧 커널 로그 분석") + if not result.kernel_events: + st.success("주요 커널 에러가 감지되지 않았습니다.") + else: + kc1, kc2, kc3, kc4 = st.columns(4) + kc1.metric("Critical", result.kernel_summary.get("critical", 0)) + kc2.metric("Error", result.kernel_summary.get("error", 0)) + kc3.metric("OOM", result.kernel_summary.get("oom", 0)) + kc4.metric("PCIe", result.kernel_summary.get("pcie_error", 0)) + + event_data = [{"시각": e.timestamp, "유형": e.level.upper(), "키워드": e.message, + "내용": e.raw_line[:200]} for e in result.kernel_events[:100]] + st.dataframe(pd.DataFrame(event_data), use_container_width=True, hide_index=True) + + # 원본 로그 + st.markdown("---") + st.subheader("📄 원본 로그 검색") + search_term = st.text_input("키워드 검색", placeholder="예: Xid, error, temperature...") + log_text = read_log_file(file_content, uploaded_file.name) + if search_term: + matched = [f"**L{i+1}:** `{ln.strip()}`" + for i, ln in enumerate(log_text.splitlines()) + if search_term.lower() in ln.lower()] + st.write(f"**{len(matched)}건 발견** (처음 50건)") + for ln in matched[:50]: + st.markdown(ln) + else: + st.text_area("로그 미리보기 (5000자)", log_text[:5000], height=200) + + +# ============================================= +# 페이지 2: 분석 이력 +# ============================================= + +def page_history(): + st.markdown('

📚 분석 이력

', unsafe_allow_html=True) + st.markdown('

이전에 분석한 모든 로그 결과를 조회하고 관리합니다.

', + unsafe_allow_html=True) + + # 필터 + fc1, fc2 = st.columns([1, 3]) + with fc1: + status_filter = st.selectbox("상태 필터", ["전체", "critical", "warning", "healthy"]) + with fc2: + search = st.text_input("검색 (파일명, 메모, 드라이버 버전)", placeholder="키워드 입력...") + + records = get_analysis_list(limit=200, status_filter=status_filter, search=search or None) + + if not records: + st.info("저장된 분석 이력이 없습니다. 로그 파일을 업로드하면 자동으로 저장됩니다.") + return + + # 요약 통계 + summary = get_status_summary() + sc1, sc2, sc3, sc4 = st.columns(4) + sc1.metric("전체 보고서", summary["total_reports"]) + sc2.metric("🔴 심각", summary["by_status"].get("critical", 0)) + sc3.metric("🟡 경고", summary["by_status"].get("warning", 0)) + sc4.metric("🟢 정상", summary["by_status"].get("healthy", 0)) + + st.markdown("---") + + # 이력 테이블 + status_emoji = {"critical": "🔴", "warning": "🟡", "healthy": "🟢"} + table_data = [{ + "ID": r["id"], + "분석일시": r["created_at"], + "상태": f"{status_emoji.get(r['overall_status'], '')} {r['overall_status']}", + "파일명": r["filename"][:40], + "GPU 수": r["gpu_count"], + "XID": r["xid_count"], + "심각": r["critical_count"], + "경고": r["warning_count"], + "드라이버": r["driver_version"], + "메모": (r.get("memo") or "")[:50], + } for r in records] + + st.dataframe(pd.DataFrame(table_data), use_container_width=True, hide_index=True) + + # 상세 조회 + st.markdown("---") + st.subheader("🔎 상세 조회") + + # 보고서 선택을 돕는 UI 개선: 드롭다운 선택박스 제공 + history_options = [ + (r["id"], f"[ID: {r['id']}] {status_emoji.get(r['overall_status'], '')} {r['filename'][:30]} ({r['created_at']})") + for r in records + ] + + selected_option = st.selectbox( + "조회할 보고서 선택", + options=history_options, + format_func=lambda x: x[1] + ) + + selected_id = selected_option[0] if selected_option else records[0]["id"] + + detail = get_analysis_detail(selected_id) + if not detail: + st.warning(f"ID {selected_id}에 해당하는 분석 결과가 없습니다.") + return + + # 메모 편집 (editor/admin만) + perms = get_perms() + current_memo = detail.get("memo", "") or "" + if perms["can_edit_memo"]: + new_memo = st.text_area("📝 메모 수정", value=current_memo, key=f"memo_{selected_id}") + if new_memo != current_memo: + update_memo(selected_id, new_memo) + st.success("메모가 업데이트되었습니다.") + elif current_memo: + st.text_area("📝 메모", value=current_memo, key=f"memo_{selected_id}", disabled=True) + + # 삭제 버튼 (admin만) + if perms["can_delete"]: + if st.button("🗑️ 이 분석 기록 삭제", type="secondary"): + delete_analysis(selected_id) + st.success(f"ID {selected_id} 삭제 완료. 페이지를 새로고침하세요.") + return + + # 상세 내용 + render_status_badge(detail["overall_status"]) + + dc1, dc2, dc3, dc4 = st.columns(4) + dc1.metric("파일명", detail["filename"][:30]) + dc2.metric("크기", f"{detail['file_size_mb']} MB") + dc3.metric("드라이버", detail["driver_version"]) + dc4.metric("CUDA", detail["cuda_version"]) + + # GPU 상세 + st.markdown("---") + st.subheader("🎮 GPU 상태") + render_gpu_details_from_dict(detail["gpus"]) + + # RMA 판정 + if detail.get("rma_verdicts"): + st.markdown("---") + render_rma_section(detail["rma_verdicts"], source="db") + + # XID 이벤트 + if detail["xid_events"]: + st.markdown("---") + st.subheader(f"⚠️ XID 이벤트 ({len(detail['xid_events'])}건)") + xid_df = pd.DataFrame([{ + "XID": x["xid_code"], "이름": x["xid_name"], "심각도": x["severity"], + "GPU": x.get("gpu_name") or "N/A", + "S/N": x.get("gpu_serial") or "N/A", + "PCI": x["pci_address"], "시각": x["timestamp"], + } for x in detail["xid_events"]]) + st.dataframe(xid_df, use_container_width=True, hide_index=True) + + # 커널 이벤트 + if detail["kernel_events"]: + st.markdown("---") + st.subheader(f"🐧 커널 이벤트 ({len(detail['kernel_events'])}건)") + ke_df = pd.DataFrame([{ + "유형": k["level"].upper(), "키워드": k["message"], + "시각": k["timestamp"], "내용": k["raw_line"][:150], + } for k in detail["kernel_events"][:100]]) + st.dataframe(ke_df, use_container_width=True, hide_index=True) + + +# ============================================= +# 페이지 3: XID 통계 +# ============================================= + +def page_xid_stats(): + st.markdown('

📊 XID 에러 통계

', unsafe_allow_html=True) + st.markdown('

전체 분석 이력에서 XID 에러 코드별 발생 빈도와 트렌드를 확인합니다.

', + unsafe_allow_html=True) + + stats = get_xid_statistics() + if not stats: + st.info("아직 XID 에러 데이터가 없습니다. 로그를 업로드하면 자동으로 축적됩니다.") + return + + # 전체 통계 테이블 + sev_emoji = {"critical": "🔴", "warning": "🟡", "info": "🟢"} + stat_data = [{ + "XID": s["xid_code"], + "이름": s["xid_name"], + "심각도": f"{sev_emoji.get(s['severity'], '')} {s['severity']}", + "총 발생": s["total_count"], + "관련 보고서": s["affected_reports"], + "최초 발견": s["first_seen"], + "최근 발견": s["last_seen"], + } for s in stats] + + st.dataframe(pd.DataFrame(stat_data), use_container_width=True, hide_index=True) + + # XID 코드별 상세 조회 + st.markdown("---") + st.subheader("🔍 XID 코드 상세 조회") + xid_codes = [s["xid_code"] for s in stats] + if xid_codes: + selected_xid = st.selectbox("XID 코드 선택", xid_codes, + format_func=lambda x: f"XID {x}: {XID_DB['xid_errors'].get(str(x), {}).get('name', 'Unknown')}") + xid_info = XID_DB["xid_errors"].get(str(selected_xid)) + if xid_info: + sev = XID_DB["severity_levels"].get(xid_info["severity"], {}) + st.markdown(f"### {sev.get('emoji', '')} XID {selected_xid}: {xid_info['name']}") + st.markdown(f"**설명:** {xid_info['description']}") + st.markdown("**권장 조치:**") + st.markdown(xid_info["action"]) + + # 전체 XID DB 조회 + st.markdown("---") + st.subheader("📖 XID 코드 사전") + xid_lookup = st.number_input("XID 코드 입력", min_value=1, max_value=999, value=61, step=1) + xid_info = XID_DB["xid_errors"].get(str(xid_lookup)) + if xid_info: + sev = XID_DB["severity_levels"].get(xid_info["severity"], {}) + st.markdown(f"**{sev.get('emoji', '')} XID {xid_lookup}: {xid_info['name']}**") + st.markdown(f"심각도: {sev.get('label', xid_info['severity'])}") + st.markdown(xid_info['description']) + st.markdown("**권장 조치:**") + st.markdown(xid_info["action"]) + else: + st.warning(f"XID {xid_lookup}에 대한 정보가 데이터베이스에 없습니다.") + + +# ============================================= +# 페이지 4: GPU 시리얼 추적 +# ============================================= + +def page_gpu_tracking(): + st.markdown('

🔧 GPU 시리얼 추적

', unsafe_allow_html=True) + st.markdown('

특정 GPU의 장애 이력을 시리얼 번호로 추적합니다. 동일 GPU의 반복 장애 패턴을 파악할 수 있습니다.

', + unsafe_allow_html=True) + + serials = get_all_gpu_serials() + if not serials: + st.info("아직 GPU 시리얼 데이터가 없습니다. 로그를 업로드하면 자동으로 축적됩니다.") + return + + # 전체 GPU 목록 + st.subheader("📋 등록된 GPU 목록") + gpu_table = [{ + "시리얼 번호": s["serial_number"], + "GPU 모델": s["name"], + "UUID": s["gpu_uuid"], + "분석 보고서 수": s["report_count"], + } for s in serials] + st.dataframe(pd.DataFrame(gpu_table), use_container_width=True, hide_index=True) + + # 시리얼 선택 + st.markdown("---") + st.subheader("🔎 시리얼 번호별 이력 조회") + serial_options = [s["serial_number"] for s in serials] + selected_serial = st.selectbox( + "시리얼 번호 선택", + serial_options, + format_func=lambda x: f"{x} ({next((s['name'] for s in serials if s['serial_number']==x), 'N/A')})" + ) + + if selected_serial: + history = get_gpu_serial_history(selected_serial) + if history: + st.write(f"**{selected_serial}** 의 분석 이력: **{len(history)}건**") + hist_data = [{ + "ID": h["id"], + "분석일시": h["created_at"], + "파일명": h["filename"][:30], + "상태": f"{'🔴' if h['overall_status']=='critical' else '🟡' if h['overall_status']=='warning' else '🟢'} {h['overall_status']}", + "이슈": h["total_issues"], + "심각": h["critical_count"], + "온도": f"{h['temperature']}°C", + "전력": f"{h['power_draw']}W", + "메모리": f"{h['memory_used']}/{h['memory_total']} MiB", + "메모": (h.get("memo") or "")[:40], + } for h in history] + st.dataframe(pd.DataFrame(hist_data), use_container_width=True, hide_index=True) + + # 온도 추이 차트 + if len(history) > 1: + st.markdown("**온도 추이**") + chart_data = pd.DataFrame({ + "분석일시": [h["created_at"] for h in history], + "온도(°C)": [h["temperature"] for h in history], + }) + st.line_chart(chart_data.set_index("분석일시")) + else: + st.info("해당 시리얼의 이력이 없습니다.") + + +# ============================================= +# 메인 - 사이드바 네비게이션 +# ============================================= + +def page_my_account(): + """내 계정 정보 및 비밀번호 변경""" + st.markdown('

🔐 내 계정

', unsafe_allow_html=True) + user = get_current_user() + if not user: + return + + role_info = ROLES.get(user["role"], ROLES["viewer"]) + st.write(f"**아이디:** `{user['username']}`") + st.write(f"**이름:** {user['display_name']}") + st.write(f"**역할:** {role_info['label']} ({user['role']})") + st.write(f"**가입일:** {user['created_at']}") + st.write(f"**최근 로그인:** {user.get('last_login') or '-'}") + + st.markdown("---") + st.subheader("비밀번호 변경") + with st.form("change_pw"): + cur_pw = st.text_input("현재 비밀번호", type="password") + new_pw = st.text_input("새 비밀번호 (6자 이상)", type="password") + new_pw2 = st.text_input("새 비밀번호 확인", type="password") + if st.form_submit_button("변경"): + auth_result = authenticate(user["username"], cur_pw) + if auth_result is None or "error" in auth_result: + st.error("현재 비밀번호가 올바르지 않습니다.") + elif new_pw != new_pw2: + st.error("새 비밀번호가 일치하지 않습니다.") + elif update_password(user["id"], new_pw): + st.success("비밀번호가 변경되었습니다.") + else: + st.error("비밀번호는 6자 이상이어야 합니다.") + + +def main(): + # 로그인 안 되었으면 로그인 페이지만 표시 + if not get_current_user(): + page_login() + return + + user = get_current_user() + perms = get_perms() + role_info = ROLES.get(user["role"], ROLES["viewer"]) + + with st.sidebar: + st.markdown("### 🖥️ NV-Log Analyzer") + st.markdown(f"**{user['display_name']}** ({role_info['label']})") + + # 메뉴 구성 (역할별) + menu_items = [] + if perms["can_upload"]: + menu_items.append("🔍 로그 분석") + menu_items.extend(["📚 분석 이력", "📊 XID 통계", "🔧 GPU 추적"]) + if perms["can_manage_users"]: + menu_items.append("👤 사용자 관리") + menu_items.append("🔐 내 계정") + + page = st.radio("메뉴", menu_items, label_visibility="collapsed") + + st.markdown("---") + if st.button("🚪 로그아웃", use_container_width=True): + del st.session_state["user"] + st.rerun() + + st.markdown( + "NV-Log Analyzer v3.0
" + "로그인 + 역할 기반 접근제어
" + "nvidia-bug-report.log 전용", + unsafe_allow_html=True, + ) + + if page == "🔍 로그 분석": + page_analyze() + elif page == "📚 분석 이력": + page_history() + elif page == "📊 XID 통계": + page_xid_stats() + elif page == "🔧 GPU 추적": + page_gpu_tracking() + elif page == "👤 사용자 관리": + page_user_management() + elif page == "🔐 내 계정": + page_my_account() + + +if __name__ == "__main__": + main() diff --git a/auth.py b/auth.py new file mode 100644 index 0000000..1c1450b --- /dev/null +++ b/auth.py @@ -0,0 +1,204 @@ +""" +NV-Log Analyzer - 인증 및 사용자 관리 모듈 +역할: admin(관리자), editor(편집자), viewer(뷰어) +""" + +import hashlib +import secrets +import sqlite3 +from contextlib import contextmanager +from pathlib import Path + +DB_PATH = Path(__file__).parent / "data" / "nv_log_history.db" + +ROLES = { + "admin": { + "label": "관리자", + "description": "모든 기능 사용 가능 + 사용자 관리", + "can_upload": True, + "can_view": True, + "can_edit_memo": True, + "can_delete": True, + "can_manage_users": True, + }, + "editor": { + "label": "편집자", + "description": "로그 업로드, 분석 결과 조회/수정 가능", + "can_upload": True, + "can_view": True, + "can_edit_memo": True, + "can_delete": False, + "can_manage_users": False, + }, + "viewer": { + "label": "뷰어", + "description": "분석 결과 조회만 가능", + "can_upload": False, + "can_view": True, + "can_edit_memo": False, + "can_delete": False, + "can_manage_users": False, + }, +} + + +@contextmanager +def get_conn(): + conn = sqlite3.connect(str(DB_PATH)) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA foreign_keys=ON") + try: + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + +def _hash_password(password: str, salt: str = None) -> tuple: + """비밀번호 해싱 (SHA-256 + salt)""" + if salt is None: + salt = secrets.token_hex(16) + hashed = hashlib.sha256((salt + password).encode()).hexdigest() + return hashed, salt + + +def init_user_db(): + """사용자 테이블 초기화 + 기본 admin 계정 생성""" + with get_conn() as conn: + conn.executescript(""" + CREATE TABLE IF NOT EXISTS users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + username TEXT NOT NULL UNIQUE, + password TEXT NOT NULL, + salt TEXT NOT NULL, + display_name TEXT DEFAULT '', + role TEXT NOT NULL DEFAULT 'viewer', + status TEXT NOT NULL DEFAULT 'pending', + created_at TEXT NOT NULL DEFAULT (datetime('now','localtime')), + approved_by TEXT DEFAULT NULL, + last_login TEXT DEFAULT NULL + ); + """) + + # 기본 admin 계정 (최초 1회만) + existing = conn.execute("SELECT id FROM users WHERE username = 'admin'").fetchone() + if not existing: + hashed, salt = _hash_password("admin1234") + conn.execute(""" + INSERT INTO users (username, password, salt, display_name, role, status) + VALUES (?, ?, ?, ?, 'admin', 'approved') + """, ("admin", hashed, salt, "시스템 관리자")) + + +def authenticate(username: str, password: str) -> dict: + """로그인 인증. 성공 시 사용자 정보 dict 반환, 실패 시 None (대소문자 무시 체크)""" + with get_conn() as conn: + user = conn.execute( + "SELECT * FROM users WHERE LOWER(username) = LOWER(?)", (username,) + ).fetchone() + + if not user: + return None + + hashed, _ = _hash_password(password, user["salt"]) + if hashed != user["password"]: + return None + + if user["status"] != "approved": + return {"error": "pending", "status": user["status"]} + + # 마지막 로그인 시간 업데이트 + conn.execute( + "UPDATE users SET last_login = datetime('now','localtime') WHERE id = ?", + (user["id"],) + ) + + return dict(user) + + +def register_user(username: str, password: str, display_name: str) -> dict: + """회원가입. 성공 시 {"success": True}, 실패 시 에러 메시지 (대소문자 무시 중복 검사)""" + if len(username) < 3: + return {"success": False, "error": "아이디는 3자 이상이어야 합니다."} + if len(password) < 6: + return {"success": False, "error": "비밀번호는 6자 이상이어야 합니다."} + + with get_conn() as conn: + existing = conn.execute( + "SELECT id FROM users WHERE LOWER(username) = LOWER(?)", (username,) + ).fetchone() + if existing: + return {"success": False, "error": "이미 사용 중인 아이디입니다."} + + hashed, salt = _hash_password(password) + conn.execute(""" + INSERT INTO users (username, password, salt, display_name, role, status) + VALUES (?, ?, ?, ?, 'viewer', 'pending') + """, (username, hashed, salt, display_name)) + + return {"success": True} + + +def get_pending_users() -> list: + """승인 대기 중인 사용자 목록""" + with get_conn() as conn: + return [dict(r) for r in conn.execute( + "SELECT * FROM users WHERE status = 'pending' ORDER BY created_at DESC" + ).fetchall()] + + +def get_all_users() -> list: + """전체 사용자 목록""" + with get_conn() as conn: + return [dict(r) for r in conn.execute( + "SELECT * FROM users ORDER BY CASE status WHEN 'pending' THEN 0 ELSE 1 END, created_at DESC" + ).fetchall()] + + +def approve_user(user_id: int, role: str, approved_by: str): + """사용자 승인 + 역할 지정""" + with get_conn() as conn: + conn.execute( + "UPDATE users SET status = 'approved', role = ?, approved_by = ? WHERE id = ?", + (role, approved_by, user_id) + ) + + +def reject_user(user_id: int): + """사용자 가입 거절""" + with get_conn() as conn: + conn.execute("UPDATE users SET status = 'rejected' WHERE id = ?", (user_id,)) + + +def update_user_role(user_id: int, new_role: str): + """사용자 역할 변경""" + with get_conn() as conn: + conn.execute("UPDATE users SET role = ? WHERE id = ?", (new_role, user_id)) + + +def delete_user(user_id: int): + """사용자 삭제""" + with get_conn() as conn: + conn.execute("DELETE FROM users WHERE id = ?", (user_id,)) + + +def update_password(user_id: int, new_password: str) -> bool: + """비밀번호 변경""" + if len(new_password) < 6: + return False + hashed, salt = _hash_password(new_password) + with get_conn() as conn: + conn.execute( + "UPDATE users SET password = ?, salt = ? WHERE id = ?", + (hashed, salt, user_id) + ) + return True + + +def get_user_permissions(role: str) -> dict: + """역할별 권한 반환""" + return ROLES.get(role, ROLES["viewer"]) diff --git a/data/xid_database.json b/data/xid_database.json new file mode 100644 index 0000000..d355102 --- /dev/null +++ b/data/xid_database.json @@ -0,0 +1,1056 @@ +{ + "xid_errors": { + "8": { + "name": "GPU Stopped Responding", + "severity": "critical", + "description": "GPU가 응답을 중단했습니다. 하드웨어 결함 또는 드라이버 문제일 수 있습니다.", + "action": "1) GPU 드라이버를 최신 버전으로 업데이트\n2) GPU 전원 케이블 재연결 확인\n3) PCIe 슬롯 변경 후 테스트\n4) 반복 시 RMA(교체) 요청" + }, + "13": { + "name": "Graphics Engine Exception", + "severity": "critical", + "description": "그래픽 엔진에서 예외가 발생했습니다. GPU 코어 또는 메모리 결함 가능성.", + "action": "1) GPU 드라이버 재설치\n2) GPU 클럭 주파수를 기본값으로 리셋\n3) 시스템 BIOS에서 PCIe Gen 설정 확인\n4) 반복 시 GPU 교체 검토" + }, + "31": { + "name": "GPU Memory Page Fault", + "severity": "critical", + "description": "GPU 메모리 페이지 폴트. 메모리 손상 또는 드라이버 버그 가능성.", + "action": "1) ECC 메모리 상태 확인 (nvidia-smi -q -d ECC)\n2) GPU 드라이버 업데이트\n3) CUDA 버전 호환성 확인\n4) ECC 에러 누적 시 GPU 메모리 교체/RMA" + }, + "32": { + "name": "Invalid or Corrupted Push Buffer", + "severity": "warning", + "description": "손상된 푸시 버퍼 감지. 드라이버 또는 애플리케이션 문제.", + "action": "1) 실행 중인 GPU 워크로드 확인\n2) 드라이버 버전 업데이트\n3) 특정 애플리케이션에서만 발생 시 해당 앱 업데이트" + }, + "38": { + "name": "Driver Firmware Mismatch", + "severity": "warning", + "description": "드라이버와 펌웨어 버전 불일치.", + "action": "1) nvidia-smi로 드라이버/펌웨어 버전 확인\n2) GPU 드라이버를 펌웨어와 호환되는 버전으로 업데이트\n3) 필요 시 GPU 펌웨어 업데이트" + }, + "43": { + "name": "GPU Stopped Processing", + "severity": "critical", + "description": "GPU 처리가 중단됨. 심각한 하드웨어 또는 드라이버 문제.", + "action": "1) 시스템 재부팅 후 재현 여부 확인\n2) dmesg에서 관련 커널 에러 확인\n3) GPU 온도 및 전력 상태 점검\n4) 벤더사 기술지원 요청 권장" + }, + "45": { + "name": "GPU Preemption Failure", + "severity": "warning", + "description": "GPU 선점 실패. 특정 워크로드에서 타임아웃 발생.", + "action": "1) 실행 중인 GPU 컴퓨트 작업 확인\n2) TDR(Timeout Detection and Recovery) 설정 조정\n3) 드라이버 업데이트" + }, + "48": { + "name": "Double Bit ECC Error", + "severity": "critical", + "description": "수정 불가능한 Double Bit ECC 에러 발생. GPU 메모리 물리적 손상.", + "action": "1) nvidia-smi -q -d ECC로 에러 위치 확인\n2) GPU 리셋 시도 (nvidia-smi -r)\n3) 즉시 RMA(교체) 요청 - 하드웨어 결함 확정" + }, + "61": { + "name": "GPU Fallen Off the Bus", + "severity": "critical", + "description": "GPU가 PCIe 버스에서 분리됨. 전원 또는 물리적 연결 문제.", + "action": "1) GPU 전원 케이블(6핀/8핀) 재연결\n2) PCIe 슬롯 물리적 접촉 상태 확인\n3) PSU 전력 용량 점검\n4) 다른 PCIe 슬롯에서 테스트\n5) 반복 시 라이저 카드/메인보드 점검" + }, + "62": { + "name": "GPU ECC Page Retirement", + "severity": "warning", + "description": "ECC 에러로 인한 메모리 페이지 은퇴 처리.", + "action": "1) nvidia-smi -q -d RETIRED_PAGES로 은퇴 페이지 수 확인\n2) 은퇴 페이지가 임계값 이상이면 GPU 교체 검토\n3) 정기적으로 모니터링 필요" + }, + "63": { + "name": "ECC Page Retirement: Double Bit", + "severity": "critical", + "description": "Double Bit ECC 에러로 인한 페이지 은퇴. 심각한 메모리 손상.", + "action": "1) 즉시 GPU 워크로드 중단\n2) nvidia-smi -q -d RETIRED_PAGES 확인\n3) GPU RMA(교체) 요청 필수" + }, + "64": { + "name": "ECC Page Retirement Failure", + "severity": "critical", + "description": "ECC 에러 페이지 은퇴 실패. 더 이상 은퇴할 수 있는 페이지가 없음.", + "action": "1) GPU 즉시 교체 필요\n2) 데이터 무결성 확인\n3) 벤더사 긴급 RMA 요청" + }, + "69": { + "name": "GPU Access to System Memory Failed", + "severity": "warning", + "description": "GPU에서 시스템 메모리 접근 실패. IOMMU 또는 드라이버 문제.", + "action": "1) BIOS에서 IOMMU/VT-d 설정 확인\n2) 드라이버 업데이트\n3) 시스템 RAM 에러 여부 확인 (memtest)" + }, + "79": { + "name": "GPU Fallen Off the Bus (Fatal)", + "severity": "critical", + "description": "GPU가 PCIe 버스에서 완전히 분리됨. Xid 61의 치명적 버전.", + "action": "1) 시스템 즉시 셧다운\n2) GPU 물리적 장착 상태 및 전원 케이블 점검\n3) PCIe 슬롯/라이저 카드 교체 테스트\n4) 벤더사 현장 점검 요청" + }, + "92": { + "name": "High Single-bit ECC Error Rate", + "severity": "warning", + "description": "단일 비트 ECC 에러 빈도가 높음. 메모리 노후화 징후.", + "action": "1) ECC 에러 카운트 모니터링\n2) GPU 메모리 클럭 다운 테스트\n3) 에러율 증가 추세 시 선제적 GPU 교체 검토" + }, + "94": { + "name": "Contained ECC Error", + "severity": "info", + "description": "격리된 ECC 에러. 자동으로 수정되었으나 모니터링 필요.", + "action": "1) 정기적으로 ECC 카운터 모니터링\n2) 빈도가 증가하면 경고 단계로 상향" + }, + "95": { + "name": "Uncontained ECC Error", + "severity": "critical", + "description": "격리되지 않은 ECC 에러. 데이터 손상 가능성.", + "action": "1) 즉시 워크로드 중단\n2) GPU 리셋 및 ECC 상태 확인\n3) 데이터 무결성 검증\n4) GPU 교체 권장" + }, + "119": { + "name": "GPU Recovery Action", + "severity": "info", + "description": "GPU 복구 작업 수행됨. 일시적 에러에서 자동 복구.", + "action": "1) 빈도가 낮으면 정상 운영\n2) 자주 발생 시 드라이버 업데이트 또는 GPU 점검" + }, + "1": { + "name": "ROBUST_CHANNEL_FIFO_ERROR_FIFO_METHOD", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "2": { + "name": "ROBUST_CHANNEL_FIFO_ERROR_SW_METHOD", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "3": { + "name": "ROBUST_CHANNEL_FIFO_ERROR_UNK_METHOD", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "4": { + "name": "ROBUST_CHANNEL_FIFO_ERROR_CHANNEL_BUSY", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "5": { + "name": "ROBUST_CHANNEL_FIFO_ERROR_RUNOUT_OVERFLOW", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "6": { + "name": "ROBUST_CHANNEL_FIFO_ERROR_PARSE_ERR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "7": { + "name": "ROBUST_CHANNEL_FIFO_ERROR_PTE_ERR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "9": { + "name": "ROBUST_CHANNEL_GR_ERROR_INSTANCE", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "10": { + "name": "ROBUST_CHANNEL_GR_ERROR_SINGLE_STEP", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "11": { + "name": "ROBUST_CHANNEL_GR_ERROR_MISSING_HW", + "severity": "warning", + "description": "[Catalog] Invalid or corrupted push buffer stream", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CHECK_APP/CUDA" + }, + "12": { + "name": "ROBUST_CHANNEL_GR_ERROR_SW_METHOD", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "14": { + "name": "ROBUST_CHANNEL_FAKE_ERROR", + "severity": "info", + "description": "[Catalog] Unused", + "action": "Immediate Action: IGNORE\nInvestigatory Action: CONTACT_SUPPORT\nTrigger Conditions: Fake or injected error from userspace" + }, + "15": { + "name": "ROBUST_CHANNEL_SCANLINE_TIMEOUT", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "16": { + "name": "ROBUST_CHANNEL_VBLANK_CALLBACK_TIMEOUT", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nInvestigatory Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "17": { + "name": "ROBUST_CHANNEL_PARAMETER_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "18": { + "name": "ROBUST_CHANNEL_BUS_MASTER_TIMEOUT_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "19": { + "name": "ROBUST_CHANNEL_DISP_MISSED_NOTIFIER", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "20": { + "name": "ROBUST_CHANNEL_MPEG_ERROR_SW_METHOD", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "21": { + "name": "ROBUST_CHANNEL_ME_ERROR_SW_METHOD", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "22": { + "name": "ROBUST_CHANNEL_VP_ERROR_SW_METHOD", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "23": { + "name": "ROBUST_CHANNEL_RC_LOGGING_ENABLED", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "24": { + "name": "ROBUST_CHANNEL_GR_SEMAPHORE_TIMEOUT", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "25": { + "name": "ROBUST_CHANNEL_GR_ILLEGAL_NOTIFY", + "severity": "warning", + "description": "[Catalog] Invalid or illegal push buffer stream", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CHECK_APP/CUDA" + }, + "26": { + "name": "ROBUST_CHANNEL_FIFO_ERROR_FBISTATE_TIMEOUT", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "27": { + "name": "ROBUST_CHANNEL_VP_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "28": { + "name": "ROBUST_CHANNEL_VP2_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "29": { + "name": "ROBUST_CHANNEL_BSP_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "30": { + "name": "ROBUST_CHANNEL_BAD_ADDR_ACCESS", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "33": { + "name": "ROBUST_CHANNEL_SEC_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "34": { + "name": "ROBUST_CHANNEL_MSVLD_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "35": { + "name": "ROBUST_CHANNEL_MSPDEC_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "36": { + "name": "ROBUST_CHANNEL_MSPPP_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "37": { + "name": "ROBUST_CHANNEL_FECS_ERR_UNIMP_FIRMWARE_METHOD", + "severity": "info", + "description": "[Catalog] Driver firmware error", + "action": "Immediate Action: IGNORE\nInvestigatory Action: CHECK_APP/CUDA" + }, + "39": { + "name": "ROBUST_CHANNEL_CE0_ERROR", + "severity": "warning", + "description": "[Catalog] Copy Engine Exception", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "40": { + "name": "ROBUST_CHANNEL_CE1_ERROR", + "severity": "warning", + "description": "[Catalog] Copy Engine Exception", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "41": { + "name": "ROBUST_CHANNEL_CE2_ERROR", + "severity": "warning", + "description": "[Catalog] Copy Engine Exception", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "42": { + "name": "ROBUST_CHANNEL_VIC_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "44": { + "name": "ROBUST_CHANNEL_GR_FAULT_DURING_CTXSW", + "severity": "info", + "description": "[Catalog] Graphics Engine fault during context switch", + "action": "Immediate Action: IGNORE\nInvestigatory Action: CONTACT_SUPPORT" + }, + "46": { + "name": "ROBUST_CHANNEL_GPU_TIMEOUT_ERROR", + "severity": "info", + "description": "[Catalog] GPU stopped processing", + "action": "Immediate Action: RESET_GPU\nInvestigatory Action: CONTACT_SUPPORT" + }, + "47": { + "name": "ROBUST_CHANNEL_NVENC0_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "49": { + "name": "SILENT_RUNNING_CONSTANT_LEVEL_SET_BY_REGISTRY", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "50": { + "name": "SILENT_RUNNING_LEVEL_TRANSITION_DUE_TO_RC_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "51": { + "name": "SILENT_RUNNING_STRESS_TEST_FAILURE", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "52": { + "name": "SILENT_RUNNING_LEVEL_TRANS_DUE_TO_TEMP_RISE", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "53": { + "name": "SILENT_RUNNING_TEMP_REDUCED_CLOCKING", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "54": { + "name": "SILENT_RUNNING_PWR_REDUCED_CLOCKING", + "severity": "warning", + "description": "[Catalog] Auxiliary power is not connected to the GPU board", + "action": "Immediate Action: CHECK_MECHANICALS\nInvestigatory Action: CONTACT_SUPPORT" + }, + "55": { + "name": "SILENT_RUNNING_TEMPERATURE_READ_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "56": { + "name": "DISPLAY_CHANNEL_EXCEPTION", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "57": { + "name": "FB_LINK_TRAINING_FAILURE_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "58": { + "name": "FB_MEMORY_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "59": { + "name": "PMU_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "60": { + "name": "ROBUST_CHANNEL_SEC2_ERROR", + "severity": "warning", + "description": "[Catalog] Video processor exception", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: INVESTIGATE_SW" + }, + "65": { + "name": "ROBUST_CHANNEL_NVENC1_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "66": { + "name": "ROBUST_CHANNEL_FECS_ERR_REG_ACCESS_VIOLATION", + "severity": "info", + "description": "[Catalog] Illegal access by driver", + "action": "Immediate Action: IGNORE\nInvestigatory Action: INVESTIGATE_SW" + }, + "67": { + "name": "ROBUST_CHANNEL_FECS_ERR_VERIF_VIOLATION", + "severity": "info", + "description": "[Catalog] Illegal access by driver", + "action": "Immediate Action: IGNORE\nInvestigatory Action: CONTACT_SUPPORT" + }, + "68": { + "name": "ROBUST_CHANNEL_NVDEC0_ERROR", + "severity": "warning", + "description": "[Catalog] NVDEC0 Exception", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "70": { + "name": "ROBUST_CHANNEL_CE3_ERROR", + "severity": "warning", + "description": "[Catalog] CE3: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "71": { + "name": "ROBUST_CHANNEL_CE4_ERROR", + "severity": "warning", + "description": "[Catalog] CE4: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "72": { + "name": "ROBUST_CHANNEL_CE5_ERROR", + "severity": "warning", + "description": "[Catalog] CE5: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "73": { + "name": "ROBUST_CHANNEL_NVENC2_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "74": { + "name": "NVLINK_ERROR", + "severity": "info", + "description": "[Catalog] NVLINK Error", + "action": "Immediate Action: WORKFLOW_NVLINK_ERR\nInvestigatory Action: CONTACT_SUPPORT\nTrigger Conditions: This event is logged when the GPU detects that a problem with a connection from the GPU to another GPU or NVSwitch over NVLink. A GPU reset or node reboot is needed to clear this error. \n\nThis event may indicate a hardware failure with the link itself, or may indicate a problem with the device at the remote end of the link. For example, if a GPU fails, another GPU connected to it over NVLink may report an Xid 74 simply because the link went down as a result. \n\nThe nvidia-smi nvlink command can provide additional details on NVLink errors, and connection information on the links. \n\nIf this error is seen repeatedly and GPU reset or node reboot fails to clear the condition, contact your hardware vendor for support." + }, + "75": { + "name": "ROBUST_CHANNEL_CE6_ERROR", + "severity": "warning", + "description": "[Catalog] CE6: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "76": { + "name": "ROBUST_CHANNEL_CE7_ERROR", + "severity": "warning", + "description": "[Catalog] CE7: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "77": { + "name": "ROBUST_CHANNEL_CE8_ERROR", + "severity": "warning", + "description": "[Catalog] CE8: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "78": { + "name": "VGPU_START_ERROR", + "severity": "warning", + "description": "[Catalog] vGPU Start Error", + "action": "Immediate Action: UPDATE_SWFW\nInvestigatory Action: UPDATE_SWFW" + }, + "80": { + "name": "PBDMA_PUSHBUFFER_CRC_MISMATCH", + "severity": "warning", + "description": "[Catalog] Corrupted data sent to GPU", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CHECK_APP/CUDA" + }, + "81": { + "name": "ROBUST_CHANNEL_VGA_SUBSYSTEM_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "82": { + "name": "ROBUST_CHANNEL_NVJPG0_ERROR", + "severity": "warning", + "description": "[Catalog] NVJPG0 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "83": { + "name": "ROBUST_CHANNEL_NVDEC1_ERROR", + "severity": "warning", + "description": "[Catalog] NVDEC1 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "84": { + "name": "ROBUST_CHANNEL_NVDEC2_ERROR", + "severity": "warning", + "description": "[Catalog] NVDEC2 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "85": { + "name": "ROBUST_CHANNEL_CE9_ERROR", + "severity": "warning", + "description": "[Catalog] CE9: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "86": { + "name": "ROBUST_CHANNEL_OFA0_ERROR", + "severity": "warning", + "description": "[Catalog] OFA Exception", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "87": { + "name": "NVTELEMETRY_DRIVER_REPORT", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "88": { + "name": "ROBUST_CHANNEL_NVDEC3_ERROR", + "severity": "warning", + "description": "[Catalog] NVDEC3 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "89": { + "name": "ROBUST_CHANNEL_NVDEC4_ERROR", + "severity": "warning", + "description": "[Catalog] NVDEC4 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "90": { + "name": "LTC_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "91": { + "name": "RESERVED_XID", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "93": { + "name": "INFOROM_ERASE_LIMIT_EXCEEDED", + "severity": "info", + "description": "[Catalog] Non-fatal violation of provisioned InfoROM wear limit", + "action": "Immediate Action: IGNORE\nInvestigatory Action: CONTACT_SUPPORT\nTrigger Conditions: This event is logged when the GPU driver fails to update the InfoROM due to violation of the provisioned InfoROM wear limit that was set for the GPU using NVFlash using nvflash --=elsessionstart.\n\nIn most cases this is not indicative of a driver or flash failure, but rather the intentional use of the InfoROM wear protection feature as set by NVFlash.\n\nRecovery steps:\nThe GPU can be recovered from Xid 93 by clearing InfoROM erase limit using ./nvflash –-elsessionclear. If clearing the limit using nvflash doesn’t help, report the issue to NVIDIA." + }, + "96": { + "name": "ROBUST_CHANNEL_NVDEC5_ERROR", + "severity": "warning", + "description": "[Catalog] NVDEC5 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "97": { + "name": "ROBUST_CHANNEL_NVDEC6_ERROR", + "severity": "warning", + "description": "[Catalog] NVDEC6 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "98": { + "name": "ROBUST_CHANNEL_NVDEC7_ERROR", + "severity": "warning", + "description": "[Catalog] NVDEC7 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "99": { + "name": "ROBUST_CHANNEL_NVJPG1_ERROR", + "severity": "warning", + "description": "[Catalog] NVJPG1 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "100": { + "name": "ROBUST_CHANNEL_NVJPG2_ERROR", + "severity": "warning", + "description": "[Catalog] NVJPG2 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "101": { + "name": "ROBUST_CHANNEL_NVJPG3_ERROR", + "severity": "warning", + "description": "[Catalog] NVJPG3 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "102": { + "name": "ROBUST_CHANNEL_NVJPG4_ERROR", + "severity": "warning", + "description": "[Catalog] NVJPG4 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "103": { + "name": "ROBUST_CHANNEL_NVJPG5_ERROR", + "severity": "warning", + "description": "[Catalog] NVJPG5 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "104": { + "name": "ROBUST_CHANNEL_NVJPG6_ERROR", + "severity": "warning", + "description": "[Catalog] NVJPG6 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "105": { + "name": "ROBUST_CHANNEL_NVJPG7_ERROR", + "severity": "warning", + "description": "[Catalog] NVJPG7 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "106": { + "name": "SMBPBI_TEST_MESSAGE", + "severity": "info", + "description": "[Catalog] SMBPBI Test Message", + "action": "Immediate Action: IGNORE\nInvestigatory Action: IGNORE" + }, + "107": { + "name": "SMBPBI_TEST_MESSAGE_SILENT", + "severity": "info", + "description": "[Catalog] SMBPBI Test Message Silent", + "action": "Immediate Action: IGNORE\nInvestigatory Action: IGNORE" + }, + "108": { + "name": "NVLINK_REMOTE_TRANSLATION_ERROR", + "severity": "info", + "description": "[Catalog] Unused", + "action": "Immediate Action: IGNORE\nInvestigatory Action: XID_137_FLOW\nTrigger Conditions: N/A; Unused" + }, + "109": { + "name": "ROBUST_CHANNEL_CTXSW_TIMEOUT_ERROR", + "severity": "info", + "description": "[Catalog] Context Switch Timeout Error", + "action": "Immediate Action: RESET_GPU\nInvestigatory Action: CONTACT_SUPPORT" + }, + "110": { + "name": "SEC_FAULT_ERROR", + "severity": "info", + "description": "[Catalog] Security Fault Error", + "action": "Immediate Action: RESET_GPU\nInvestigatory Action: INVESTIGATE_SW\nTrigger Conditions: This event should be uncommon unless there is a hardware failure. To recover, revert any recent system hardware modifications and cold reset the system. If this fails to correct the issue, contact your hardware vendor for assistance." + }, + "111": { + "name": "BUNDLE_ERROR_EVENT", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "112": { + "name": "DISP_SUPERVISOR_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "113": { + "name": "DP_LT_FAILURE", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "114": { + "name": "HEAD_RG_UNDERFLOW", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "115": { + "name": "CORE_CHANNEL_REGS", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "116": { + "name": "WINDOW_CHANNEL_REGS", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "117": { + "name": "CURSOR_CHANNEL_REGS", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "118": { + "name": "HEAD_REGS", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "120": { + "name": "GSP_ERROR", + "severity": "info", + "description": "[Catalog] GSP Error", + "action": "Immediate Action: RESET_GPU\nInvestigatory Action: INVESTIGATE_SW\nTrigger Conditions: These events (119/120) may be logged when an error occurs in code running on the GSP core of the GPU and/or a timeout occurs while waiting for the GSP core of the GPU to respond to an RPC message. A GPU reset or node power cycle may be needed if the error persists. If this problem reoccurs after a power cycle, follow the NVIDIA GPU Debug Guidelines document for additional debugging steps." + }, + "121": { + "name": "C2C_ERROR", + "severity": "info", + "description": "[Catalog] C2C Error", + "action": "Immediate Action: IGNORE\nInvestigatory Action: CONTACT_SUPPORT\nTrigger Conditions: This event may occur when the GPU driver has observed corrected errors on the C2C NVLink connection to a Grace CPU. These errors are corrected by the system and have no operational impact. Resetting the GPU at an available service window will allow the GPU to retrain the link.\nNOTE: repeat errors may be reported; VBIOS 97.00.90.00.00 may provide some relief from that condition" + }, + "122": { + "name": "SPI_PMU_RPC_READ_FAIL", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "123": { + "name": "SPI_PMU_RPC_WRITE_FAIL", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "124": { + "name": "SPI_PMU_RPC_ERASE_FAIL", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "125": { + "name": "INFOROM_FS_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "126": { + "name": "ROBUST_CHANNEL_CE10_ERROR", + "severity": "warning", + "description": "[Catalog] CE10: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "127": { + "name": "ROBUST_CHANNEL_CE11_ERROR", + "severity": "warning", + "description": "[Catalog] CE11: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "128": { + "name": "ROBUST_CHANNEL_CE12_ERROR", + "severity": "warning", + "description": "[Catalog] CE12: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "129": { + "name": "ROBUST_CHANNEL_CE13_ERROR", + "severity": "warning", + "description": "[Catalog] CE13: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "130": { + "name": "ROBUST_CHANNEL_CE14_ERROR", + "severity": "warning", + "description": "[Catalog] CE14: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "131": { + "name": "ROBUST_CHANNEL_CE15_ERROR", + "severity": "warning", + "description": "[Catalog] CE15: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "132": { + "name": "ROBUST_CHANNEL_CE16_ERROR", + "severity": "warning", + "description": "[Catalog] CE16: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "133": { + "name": "ROBUST_CHANNEL_CE17_ERROR", + "severity": "warning", + "description": "[Catalog] CE17: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "134": { + "name": "ROBUST_CHANNEL_CE18_ERROR", + "severity": "warning", + "description": "[Catalog] CE18: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "135": { + "name": "ROBUST_CHANNEL_CE19_ERROR", + "severity": "warning", + "description": "[Catalog] CE19: Unknown Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "136": { + "name": "ALI_TRAINING_FAIL", + "severity": "info", + "description": "[Catalog] Link Training Failed", + "action": "Immediate Action: RESET_GPU\nInvestigatory Action: INVESTIGATE_LINK_SI" + }, + "137": { + "name": "NVLINK_PRIV_ERR", + "severity": "info", + "description": "[Catalog] NVLink Privilege Error", + "action": "Immediate Action: IGNORE\nInvestigatory Action: XID_137_FLOW\nTrigger Conditions: This event is logged when a fault is reported by the remote MMU, such as when an illegal NVLink peer-to-peer access is made by an applicable unit on the chip. Typically these are application-level bugs, but can also be driver bugs or hardware bugs.\n\nWhen this event is logged, NVIDIA recommends the following: #. Run the application in cuda-gdb or the Compute Sanitizer memcheck tool , or #. Run the application with CUDA_DEVICE_WAITS_ON_EXCEPTION=1 and then attach later with cuda-gdb, or #. File a bug if the previous two come back inconclusive to eliminate potential NVIDIA driver or hardware bug." + }, + "138": { + "name": "ROBUST_CHANNEL_DLA_ERROR", + "severity": "critical", + "description": "[Catalog] Unused", + "action": "Immediate Action: CONTACT_SUPPORT\nTrigger Conditions: N/A; Unused" + }, + "139": { + "name": "ROBUST_CHANNEL_OFA1_ERROR", + "severity": "warning", + "description": "[Catalog] OFA1 Error", + "action": "Immediate Action: RESTART_APP\nInvestigatory Action: CONTACT_SUPPORT" + }, + "140": { + "name": "UNRECOVERABLE_ECC_ERROR_ESCAPE", + "severity": "info", + "description": "[Catalog] ECC Unrecovered Error", + "action": "Immediate Action: RESET_GPU\nInvestigatory Action: CONTACT_SUPPORT\nTrigger Conditions: This event may occur when the GPU driver has observed uncorrectable errors in GPU memory, in such a way as to interrupt the GPU driver’s ability to mark the pages for dynamic page offlining or row remapping. Reset the GPU, and if the problem persists, contact your hardware vendor for support." + }, + "141": { + "name": "ROBUST_CHANNEL_FAST_PATH_ERROR", + "severity": "info", + "description": "[Catalog] CUDA Fast Path Error", + "action": "Immediate Action: IGNORE\nInvestigatory Action: CONTACT_SUPPORT" + }, + "142": { + "name": "ROBUST_CHANNEL_NVENC3_ERROR", + "severity": "critical", + "description": "[Catalog] NVENC3 Error", + "action": "Immediate Action: CONTACT_SUPPORT" + }, + "143": { + "name": "GPU_INIT_ERROR", + "severity": "info", + "description": "[Catalog] GPU Initialization Error", + "action": "Immediate Action: RESET_GPU\nInvestigatory Action: CONTACT_SUPPORT" + }, + "144": { + "name": "NVLINK_SAW_ERROR", + "severity": "info", + "description": "[Catalog] NVLINK: SAW Error", + "action": "Immediate Action: WORKFLOW_NVLINK5_ERR\nInvestigatory Action: WORKFLOW_NVLINK5_ERR" + }, + "145": { + "name": "NVLINK_RLW_ERROR", + "severity": "info", + "description": "[Catalog] NVLINK: RLW Error", + "action": "Immediate Action: WORKFLOW_NVLINK5_ERR\nInvestigatory Action: WORKFLOW_NVLINK5_ERR" + }, + "146": { + "name": "NVLINK_TLW_ERROR", + "severity": "info", + "description": "[Catalog] NVLINK: TLW Error", + "action": "Immediate Action: WORKFLOW_NVLINK5_ERR\nInvestigatory Action: WORKFLOW_NVLINK5_ERR" + }, + "147": { + "name": "NVLINK_TREX_ERROR", + "severity": "info", + "description": "[Catalog] NVLINK: TREX Error", + "action": "Immediate Action: WORKFLOW_NVLINK5_ERR\nInvestigatory Action: WORKFLOW_NVLINK5_ERR" + }, + "148": { + "name": "NVLINK_NVLPW_CTRL_ERROR", + "severity": "info", + "description": "[Catalog] NVLINK: NVLPW_CTRL Error", + "action": "Immediate Action: WORKFLOW_NVLINK5_ERR\nInvestigatory Action: WORKFLOW_NVLINK5_ERR" + }, + "149": { + "name": "NVLINK_NETIR_ERROR", + "severity": "info", + "description": "[Catalog] NVLINK: NETIR Error", + "action": "Immediate Action: WORKFLOW_NVLINK5_ERR\nInvestigatory Action: WORKFLOW_NVLINK5_ERR" + }, + "150": { + "name": "NVLINK_MSE_ERROR", + "severity": "info", + "description": "[Catalog] NVLINK: MSE Error", + "action": "Immediate Action: WORKFLOW_NVLINK5_ERR\nInvestigatory Action: WORKFLOW_NVLINK5_ERR" + }, + "151": { + "name": "ROBUST_CHANNEL_KEY_ROTATION_ERROR", + "severity": "warning", + "description": "[Catalog] Key rotation Error", + "action": "Immediate Action: RESTART_VM\nInvestigatory Action: CONTACT_SUPPORT" + }, + "152": { + "name": "ROBUST_CHANNEL_DLA_SMMU_ERROR", + "severity": "info", + "description": "[Catalog] DLA SMMU Error", + "action": "Immediate Action: IGNORE\nInvestigatory Action: CONTACT_SUPPORT" + }, + "153": { + "name": "ROBUST_CHANNEL_DLA_TIMEOUT", + "severity": "info", + "description": "[Catalog] DLA timeout Error", + "action": "Immediate Action: IGNORE\nInvestigatory Action: CONTACT_SUPPORT" + }, + "154": { + "name": "GPU_RECOVERY_ACTION_CHANGED", + "severity": "info", + "description": "[Catalog] GPU Recovery Action Changed", + "action": "Immediate Action: XID_154\nInvestigatory Action: N/A Informational only regarding another Xid\nTrigger Conditions: \"Xid 154 will be seen in conjunction with other Xids and summarizes the recovery action required for other Xids. \nThe string will be similar to \"\"Xid 154 GPU recovery action changed from 0x0 (None) to 0x2 (Node Reboot Required)\"\" where the expected values of the text are: \"\"None\"\", \"\"Drain P2P\"\", \"\"Drain and Reset\"\", \"\"GPU Reset Required\"\", \"\"Node Reboot Required\"\". \"" + }, + "155": { + "name": "NVLINK_SW_DEFINED_ERROR", + "severity": "info", + "description": "[Catalog] NVLINK: SW Defined Error", + "action": "Immediate Action: RESET_GPU\nInvestigatory Action: INVESTIGATE_SW_USER\nTrigger Conditions: Link down events which are flagged as “intentional” (including transitions to SLEEP) will trigger this Xid" + }, + "156": { + "name": "RESOURCE_RETIREMENT_EVENT", + "severity": "info", + "description": "[Catalog] Resource Retirement Event", + "action": "Immediate Action: RESET_GPU\nInvestigatory Action: IGNORE" + }, + "157": { + "name": "RESOURCE_RETIREMENT_FAILURE", + "severity": "info", + "description": "[Catalog] Resource Retirement Failure", + "action": "Immediate Action: IGNORE\nInvestigatory Action: CONTACT_SUPPORT\nTrigger Conditions: No possible repairs are possible due to lack of resources. You may still run workloads or Apps, but may experience the same Xid again." + }, + "158": { + "name": "GPU_FATAL_TIMEOUT", + "severity": "info", + "description": "[Catalog] GPU Fatal Timeout", + "action": "Immediate Action: RESET_GPU\nInvestigatory Action: CONTACT_SUPPORT" + }, + "159": { + "name": "ROBUST_CHANNEL_CHI_NON_DATA_ERROR", + "severity": "warning", + "description": "[Catalog] CHI Non-Data Error", + "action": "Immediate Action: CHECK_UVM\nInvestigatory Action: SYMPATHETIC_REPORT_SOLO\nTrigger Conditions: May be seen on any C2C link-connected GPU." + }, + "160": { + "name": "CHANNEL_RETIREMENT_EVENT", + "severity": "info", + "description": "[Catalog] Channel Retirement Event", + "action": "Immediate Action: IGNORE\nInvestigatory Action: INVESTIGATE_SW" + }, + "161": { + "name": "CHANNEL_RETIREMENT_FAILURE", + "severity": "info", + "description": "[Catalog] Channel Retirement Failure", + "action": "Immediate Action: IGNORE\nInvestigatory Action: INVESTIGATE_SW" + }, + "162": { + "name": "PSHC_REENGAGED", + "severity": "info", + "description": "[Catalog] Power Smoothing HW Circuitry capability reengaged", + "action": "NVIDIA 공식 문서에서 해당 XID 코드를 확인하세요." + }, + "163": { + "name": "PSHC_DISENGAGED", + "severity": "info", + "description": "[Catalog] Power Smoothing HW Circuitry capability disengaged", + "action": "Trigger Conditions: No GPU reset required. If power smoothing functionality is desired, the customer needs to resolve the thermal events. If disabled due to timeout, reload the driver or reset the GPU." + }, + "164": { + "name": "PSHC_LOW_LIFETIME", + "severity": "info", + "description": "[Catalog] Power Smoothing HW Circuitry low lifetime reached", + "action": "Trigger Conditions: Monitor power swings and expect to replace GPUs if power smoothing is desired.\nPower smoothing functionality will be disabled soon. Investigate if power swings are acceptable, and if not, take action." + }, + "165": { + "name": "PSHC_ZERO_LIFETIME", + "severity": "info", + "description": "[Catalog] Power Smoothing HW Circuitry lifetime exhausted", + "action": "Trigger Conditions: Replace GPUs if power swings are not acceptable, and power smoothing is desired. Power smoothing will be disabled by the driver and power swings will occur. Analyze datacenter infrastructure to ensure ability to absorb power swings." + }, + "166": { + "name": "NVLINK_SECURE_CRYPTO_ERR", + "severity": "info", + "description": "[Catalog] CC traffic seen prior to link properly being configured for encrypted traffic", + "action": "Trigger Conditions: Applicable to CC (confidential computing) mode only." + }, + "167": { + "name": "PCIE_FATAL_TIMEOUT", + "severity": "info", + "description": "[Catalog] PCIE_FATAL_TIMEOUT", + "action": "NVIDIA 공식 문서에서 해당 XID 코드를 확인하세요." + }, + "168": { + "name": "REDUCED_GPU_MEMORY_CAPACITY", + "severity": "info", + "description": "[Catalog] Errors found in WPR (write protected region)", + "action": "Trigger Conditions: Should only be seen when ECC is disabled. \nEither ECC should be enabled (to enable row-remapping) or boot re-attempted with shifted WPR. " + }, + "169": { + "name": "SEC2_HALT_ERROR", + "severity": "info", + "description": "[Catalog] Internal micro-controller halt", + "action": "NVIDIA 공식 문서에서 해당 XID 코드를 확인하세요." + }, + "170": { + "name": "NVLINK_SECURE_OTHER", + "severity": "info", + "description": "[Catalog] Interrupt seen in CC mode", + "action": "Trigger Conditions: Applicable to CC (confidential computing) mode only." + }, + "171": { + "name": "UNCORRECTABLE_DRAM_ERROR", + "severity": "info", + "description": "[Catalog] Additional to Xid 48 providing more details on particulars of fault to differentiate DRAM/SRAM", + "action": "NVIDIA 공식 문서에서 해당 XID 코드를 확인하세요." + }, + "172": { + "name": "UNCORRECTABLE_SRAM_ERROR", + "severity": "info", + "description": "[Catalog] Additional to Xid 48 providing more details on particulars of fault to differentiate DRAM/SRAM", + "action": "NVIDIA 공식 문서에서 해당 XID 코드를 확인하세요." + } + }, + "severity_levels": { + "critical": { + "label": "심각", + "color": "red", + "emoji": "🔴", + "description": "즉각적인 조치가 필요합니다. 서비스 중단 또는 하드웨어 손상 위험." + }, + "warning": { + "label": "경고", + "color": "orange", + "emoji": "🟡", + "description": "주의가 필요합니다. 방치 시 심각한 문제로 발전 가능." + }, + "info": { + "label": "정보", + "color": "green", + "emoji": "🟢", + "description": "참고 사항입니다. 정기 모니터링 권장." + } + } +} \ No newline at end of file diff --git a/db.py b/db.py new file mode 100644 index 0000000..05466c9 --- /dev/null +++ b/db.py @@ -0,0 +1,412 @@ +""" +NV-Log Analyzer - SQLite 데이터베이스 모듈 +분석 결과를 영구 저장하여 장애 케이스 이력을 축적합니다. +""" + +import json +import sqlite3 +from contextlib import contextmanager +from datetime import datetime +from pathlib import Path + +DB_PATH = Path(__file__).parent / "data" / "nv_log_history.db" + + +@contextmanager +def get_conn(): + """DB 커넥션 컨텍스트 매니저""" + conn = sqlite3.connect(str(DB_PATH)) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA foreign_keys=ON") + try: + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + +def init_db(): + """DB 테이블 초기화 (앱 시작 시 1회 호출)""" + with get_conn() as conn: + conn.executescript(""" + CREATE TABLE IF NOT EXISTS analysis ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL DEFAULT (datetime('now','localtime')), + filename TEXT NOT NULL, + file_size_mb REAL, + overall_status TEXT, + total_issues INTEGER DEFAULT 0, + critical_count INTEGER DEFAULT 0, + warning_count INTEGER DEFAULT 0, + info_count INTEGER DEFAULT 0, + driver_version TEXT, + cuda_version TEXT, + gpu_count INTEGER DEFAULT 0, + xid_count INTEGER DEFAULT 0, + kernel_event_count INTEGER DEFAULT 0, + rma_count INTEGER DEFAULT 0, + ecc_info TEXT, + memo TEXT DEFAULT '' + ); + + CREATE TABLE IF NOT EXISTS analysis_gpu ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + analysis_id INTEGER NOT NULL REFERENCES analysis(id) ON DELETE CASCADE, + gpu_index INTEGER, + name TEXT, + serial_number TEXT, + gpu_uuid TEXT, + pci_address TEXT, + product_brand TEXT, + product_architecture TEXT, + driver_version TEXT, + cuda_version TEXT, + vbios_version TEXT, + inforom_version TEXT, + temperature INTEGER, + temperature_limit TEXT, + power_draw REAL, + power_limit REAL, + power_state TEXT, + memory_used INTEGER, + memory_total INTEGER, + memory_free INTEGER, + gpu_utilization INTEGER, + memory_utilization INTEGER, + fan_speed INTEGER, + persistence_mode TEXT, + compute_mode TEXT, + mig_mode TEXT, + ecc_mode TEXT, + pci_link_gen TEXT, + pci_link_width TEXT, + clock_graphics TEXT, + clock_memory TEXT, + -- RMA 관련 + remap_corr_error INTEGER DEFAULT 0, + remap_uncorr_error INTEGER DEFAULT 0, + remap_pending TEXT, + remap_failure TEXT, + remap_bank_max INTEGER DEFAULT 0, + remap_bank_high INTEGER DEFAULT 0, + remap_bank_partial INTEGER DEFAULT 0, + remap_bank_low INTEGER DEFAULT 0, + remap_bank_none INTEGER DEFAULT 0, + sram_corr_volatile TEXT, + sram_uncorr_parity_volatile TEXT, + sram_uncorr_secded_volatile TEXT, + sram_corr_aggregate TEXT, + sram_uncorr_parity_aggregate TEXT, + sram_uncorr_secded_aggregate TEXT, + sram_threshold_exceeded TEXT, + sram_l2 TEXT, + sram_sm TEXT, + sram_microcontroller TEXT, + sram_pcie TEXT, + sram_other TEXT, + dram_corr_volatile TEXT, + dram_uncorr_volatile TEXT, + dram_corr_aggregate TEXT, + dram_uncorr_aggregate TEXT, + -- HW Slowdown + hw_slowdown TEXT, + hw_thermal_slowdown TEXT, + hw_power_brake_slowdown TEXT + ); + + CREATE TABLE IF NOT EXISTS analysis_rma ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + analysis_id INTEGER NOT NULL REFERENCES analysis(id) ON DELETE CASCADE, + gpu_index INTEGER, + gpu_name TEXT, + serial_number TEXT, + qualifies_for_rma INTEGER DEFAULT 0, + reasons TEXT, + warnings TEXT, + row_remap_details TEXT, + sram_details TEXT + ); + + CREATE TABLE IF NOT EXISTS analysis_xid ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + analysis_id INTEGER NOT NULL REFERENCES analysis(id) ON DELETE CASCADE, + timestamp TEXT, + xid_code INTEGER, + xid_name TEXT, + severity TEXT, + pci_address TEXT, + gpu_name TEXT, + gpu_serial TEXT, + description TEXT, + action TEXT, + raw_line TEXT + ); + + CREATE TABLE IF NOT EXISTS analysis_kernel ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + analysis_id INTEGER NOT NULL REFERENCES analysis(id) ON DELETE CASCADE, + timestamp TEXT, + level TEXT, + message TEXT, + raw_line TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_analysis_created ON analysis(created_at); + CREATE INDEX IF NOT EXISTS idx_analysis_status ON analysis(overall_status); + CREATE INDEX IF NOT EXISTS idx_xid_code ON analysis_xid(xid_code); + CREATE INDEX IF NOT EXISTS idx_xid_severity ON analysis_xid(severity); + CREATE INDEX IF NOT EXISTS idx_gpu_serial ON analysis_gpu(serial_number); + CREATE INDEX IF NOT EXISTS idx_kernel_level ON analysis_kernel(level); + CREATE INDEX IF NOT EXISTS idx_rma_serial ON analysis_rma(serial_number); + CREATE INDEX IF NOT EXISTS idx_rma_qualifies ON analysis_rma(qualifies_for_rma); + """) + # 기존 DB 마이그레이션: gpu_name 컬럼이 없으면 추가 + cols = {row[1] for row in conn.execute("PRAGMA table_info(analysis_xid)")} + if "gpu_name" not in cols: + conn.execute("ALTER TABLE analysis_xid ADD COLUMN gpu_name TEXT") + if "gpu_serial" not in cols: + conn.execute("ALTER TABLE analysis_xid ADD COLUMN gpu_serial TEXT") + # HW slowdown 마이그레이션 + gpu_cols = {row[1] for row in conn.execute("PRAGMA table_info(analysis_gpu)")} + if "hw_slowdown" not in gpu_cols: + conn.execute("ALTER TABLE analysis_gpu ADD COLUMN hw_slowdown TEXT") + if "hw_thermal_slowdown" not in gpu_cols: + conn.execute("ALTER TABLE analysis_gpu ADD COLUMN hw_thermal_slowdown TEXT") + if "hw_power_brake_slowdown" not in gpu_cols: + conn.execute("ALTER TABLE analysis_gpu ADD COLUMN hw_power_brake_slowdown TEXT") + + +def save_analysis(result) -> int: + """분석 결과 전체를 DB에 저장하고 analysis_id 반환""" + with get_conn() as conn: + cur = conn.execute(""" + INSERT INTO analysis + (filename, file_size_mb, overall_status, + total_issues, critical_count, warning_count, info_count, + driver_version, cuda_version, gpu_count, + xid_count, kernel_event_count, rma_count, ecc_info) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?) + """, ( + result.filename, + result.file_size_mb, + result.overall_status, + result.total_issues, + result.critical_count, + result.warning_count, + result.info_count, + result.driver_version, + result.cuda_version, + len(result.gpus), + len(result.xid_events), + len(result.kernel_events), + result.rma_count, + json.dumps(result.ecc_info, ensure_ascii=False) if result.ecc_info else "{}", + )) + analysis_id = cur.lastrowid + + # GPU 정보 저장 + for gpu in result.gpus: + conn.execute(""" + INSERT INTO analysis_gpu + (analysis_id, gpu_index, name, serial_number, gpu_uuid, + pci_address, product_brand, product_architecture, + driver_version, cuda_version, vbios_version, inforom_version, + temperature, temperature_limit, power_draw, power_limit, power_state, + memory_used, memory_total, memory_free, + gpu_utilization, memory_utilization, + fan_speed, persistence_mode, compute_mode, mig_mode, ecc_mode, + pci_link_gen, pci_link_width, clock_graphics, clock_memory, + remap_corr_error, remap_uncorr_error, remap_pending, remap_failure, + remap_bank_max, remap_bank_high, remap_bank_partial, remap_bank_low, remap_bank_none, + sram_corr_volatile, sram_uncorr_parity_volatile, sram_uncorr_secded_volatile, + sram_corr_aggregate, sram_uncorr_parity_aggregate, sram_uncorr_secded_aggregate, + sram_threshold_exceeded, + sram_l2, sram_sm, sram_microcontroller, sram_pcie, sram_other, + dram_corr_volatile, dram_uncorr_volatile, dram_corr_aggregate, dram_uncorr_aggregate, + hw_slowdown, hw_thermal_slowdown, hw_power_brake_slowdown) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, + ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + """, ( + analysis_id, gpu.index, gpu.name, gpu.serial_number, gpu.gpu_uuid, + gpu.pci_address, gpu.product_brand, gpu.product_architecture, + gpu.driver_version, gpu.cuda_version, gpu.vbios_version, gpu.inforom_version, + gpu.temperature, gpu.temperature_limit, gpu.power_draw, gpu.power_limit, gpu.power_state, + gpu.memory_used, gpu.memory_total, gpu.memory_free, + gpu.gpu_utilization, gpu.memory_utilization, + gpu.fan_speed, gpu.persistence_mode, gpu.compute_mode, gpu.mig_mode, gpu.ecc_mode, + gpu.pci_link_gen_current, gpu.pci_link_width_current, + gpu.clock_graphics, gpu.clock_memory, + gpu.remap_corr_error, gpu.remap_uncorr_error, gpu.remap_pending, gpu.remap_failure, + gpu.remap_bank_max, gpu.remap_bank_high, gpu.remap_bank_partial, gpu.remap_bank_low, gpu.remap_bank_none, + gpu.sram_corr_volatile, gpu.sram_uncorr_parity_volatile, gpu.sram_uncorr_secded_volatile, + gpu.sram_corr_aggregate, gpu.sram_uncorr_parity_aggregate, gpu.sram_uncorr_secded_aggregate, + gpu.sram_threshold_exceeded, + gpu.sram_l2, gpu.sram_sm, gpu.sram_microcontroller, gpu.sram_pcie, gpu.sram_other, + gpu.dram_corr_volatile, gpu.dram_uncorr_volatile, gpu.dram_corr_aggregate, gpu.dram_uncorr_aggregate, + gpu.hw_slowdown, gpu.hw_thermal_slowdown, gpu.hw_power_brake_slowdown, + )) + + # XID 이벤트 저장 + for xid in result.xid_events: + conn.execute(""" + INSERT INTO analysis_xid + (analysis_id, timestamp, xid_code, xid_name, severity, + pci_address, gpu_name, gpu_serial, description, action, raw_line) + VALUES (?,?,?,?,?,?,?,?,?,?,?) + """, ( + analysis_id, xid.timestamp, xid.xid_code, xid.name, xid.severity, + xid.pci_address, xid.gpu_name, xid.gpu_serial, xid.description, xid.action, xid.raw_line, + )) + + # 커널 이벤트 저장 + for ke in result.kernel_events: + conn.execute(""" + INSERT INTO analysis_kernel + (analysis_id, timestamp, level, message, raw_line) + VALUES (?,?,?,?,?) + """, ( + analysis_id, ke.timestamp, ke.level, ke.message, ke.raw_line, + )) + + # RMA 판정 결과 저장 + for v in result.rma_verdicts: + conn.execute(""" + INSERT INTO analysis_rma + (analysis_id, gpu_index, gpu_name, serial_number, + qualifies_for_rma, reasons, warnings, + row_remap_details, sram_details) + VALUES (?,?,?,?,?,?,?,?,?) + """, ( + analysis_id, v.gpu_index, v.gpu_name, v.serial_number, + 1 if v.qualifies_for_rma else 0, + json.dumps(v.reasons, ensure_ascii=False), + json.dumps(v.warnings, ensure_ascii=False), + json.dumps(v.row_remap_details, ensure_ascii=False), + json.dumps(v.sram_details, ensure_ascii=False), + )) + + return analysis_id + + +def update_memo(analysis_id: int, memo: str): + """분석 건에 메모 추가/수정""" + with get_conn() as conn: + conn.execute("UPDATE analysis SET memo = ? WHERE id = ?", (memo, analysis_id)) + + +def delete_analysis(analysis_id: int): + """분석 건 삭제 (CASCADE로 하위 테이블도 삭제)""" + with get_conn() as conn: + conn.execute("DELETE FROM analysis WHERE id = ?", (analysis_id,)) + + +def get_analysis_list(limit=100, status_filter=None, search=None): + """분석 이력 목록 조회""" + with get_conn() as conn: + query = "SELECT * FROM analysis WHERE 1=1" + params = [] + if status_filter and status_filter != "전체": + query += " AND overall_status = ?" + params.append(status_filter) + if search: + query += " AND (filename LIKE ? OR memo LIKE ? OR driver_version LIKE ?)" + params.extend([f"%{search}%", f"%{search}%", f"%{search}%"]) + query += " ORDER BY created_at DESC LIMIT ?" + params.append(limit) + return [dict(row) for row in conn.execute(query, params).fetchall()] + + +def get_analysis_detail(analysis_id: int): + """분석 건 상세 조회 (GPU, XID, 커널 이벤트 포함)""" + with get_conn() as conn: + analysis = conn.execute("SELECT * FROM analysis WHERE id = ?", (analysis_id,)).fetchone() + if not analysis: + return None + result = dict(analysis) + result["gpus"] = [dict(r) for r in conn.execute( + "SELECT * FROM analysis_gpu WHERE analysis_id = ? ORDER BY gpu_index", (analysis_id,) + ).fetchall()] + result["xid_events"] = [dict(r) for r in conn.execute( + "SELECT * FROM analysis_xid WHERE analysis_id = ? ORDER BY id", (analysis_id,) + ).fetchall()] + result["kernel_events"] = [dict(r) for r in conn.execute( + "SELECT * FROM analysis_kernel WHERE analysis_id = ? ORDER BY id", (analysis_id,) + ).fetchall()] + rma_rows = conn.execute( + "SELECT * FROM analysis_rma WHERE analysis_id = ? ORDER BY gpu_index", (analysis_id,) + ).fetchall() + result["rma_verdicts"] = [] + for r in rma_rows: + d = dict(r) + d["reasons"] = json.loads(d.get("reasons") or "[]") + d["warnings"] = json.loads(d.get("warnings") or "[]") + d["row_remap_details"] = json.loads(d.get("row_remap_details") or "{}") + d["sram_details"] = json.loads(d.get("sram_details") or "{}") + result["rma_verdicts"].append(d) + return result + + +def get_xid_statistics(): + """전체 이력에서 XID 코드별 발생 통계""" + with get_conn() as conn: + return [dict(r) for r in conn.execute(""" + SELECT xid_code, xid_name, severity, + COUNT(*) as total_count, + COUNT(DISTINCT analysis_id) as affected_reports, + MIN(a.created_at) as first_seen, + MAX(a.created_at) as last_seen + FROM analysis_xid x + JOIN analysis a ON a.id = x.analysis_id + GROUP BY xid_code + ORDER BY total_count DESC + """).fetchall()] + + +def get_gpu_serial_history(serial_number: str): + """특정 GPU 시리얼 번호의 장애 이력 추적""" + with get_conn() as conn: + return [dict(r) for r in conn.execute(""" + SELECT a.id, a.created_at, a.filename, a.overall_status, + a.total_issues, a.critical_count, a.memo, + g.temperature, g.power_draw, g.memory_used, g.memory_total + FROM analysis_gpu g + JOIN analysis a ON a.id = g.analysis_id + WHERE g.serial_number = ? + ORDER BY a.created_at DESC + """, (serial_number,)).fetchall()] + + +def get_all_gpu_serials(): + """DB에 기록된 모든 GPU 시리얼 번호 목록""" + with get_conn() as conn: + return [dict(r) for r in conn.execute(""" + SELECT DISTINCT serial_number, name, gpu_uuid, + COUNT(DISTINCT analysis_id) as report_count + FROM analysis_gpu + WHERE serial_number != 'N/A' + GROUP BY serial_number + ORDER BY report_count DESC + """).fetchall()] + + +def get_status_summary(): + """전체 분석 이력 상태 요약 (대시보드용)""" + with get_conn() as conn: + total = conn.execute("SELECT COUNT(*) as cnt FROM analysis").fetchone()["cnt"] + by_status = {r["overall_status"]: r["cnt"] for r in conn.execute( + "SELECT overall_status, COUNT(*) as cnt FROM analysis GROUP BY overall_status" + ).fetchall()} + recent_critical = [dict(r) for r in conn.execute(""" + SELECT id, created_at, filename, critical_count, memo + FROM analysis WHERE overall_status = 'critical' + ORDER BY created_at DESC LIMIT 5 + """).fetchall()] + return { + "total_reports": total, + "by_status": by_status, + "recent_critical": recent_critical, + } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..dfb3344 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +version: "3.8" + +services: + nv-log-analyzer: + build: . + container_name: nv-log-analyzer + ports: + - "8501:8501" + restart: unless-stopped + volumes: + - ./data:/app/data + environment: + - STREAMLIT_SERVER_MAX_UPLOAD_SIZE=500 diff --git a/log_parser.py b/log_parser.py new file mode 100644 index 0000000..0491612 --- /dev/null +++ b/log_parser.py @@ -0,0 +1,997 @@ +""" +NV-Log Analyzer - 핵심 분석 엔진 +nvidia-bug-report.log.gz 파일을 파싱하여 XID 에러, GPU 상태, 커널 로그를 분석합니다. +성능 최적화: 모든 파싱을 라인 단위로 처리하여 대용량 파일에서도 빠르게 동작합니다. +""" + +import gzip +import json +import re +from collections import Counter, defaultdict +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path + + +# XID 데이터베이스 로드 +DATA_DIR = Path(__file__).parent / "data" +with open(DATA_DIR / "xid_database.json", "r", encoding="utf-8") as f: + XID_DB = json.load(f) + + +# --- 데이터 클래스 --- + +@dataclass +class XidEvent: + timestamp: str + xid_code: int + pci_address: str + raw_line: str + name: str = "" + severity: str = "info" + description: str = "" + action: str = "" + gpu_index: int = -1 + gpu_name: str = "" + gpu_serial: str = "" + + def __post_init__(self): + xid_info = XID_DB["xid_errors"].get(str(self.xid_code)) + if xid_info: + self.name = xid_info["name"] + self.severity = xid_info["severity"] + self.description = xid_info["description"] + self.action = xid_info["action"] + else: + self.name = f"Unknown XID {self.xid_code}" + self.severity = "warning" + self.description = f"XID {self.xid_code}에 대한 정보가 데이터베이스에 없습니다." + self.action = "NVIDIA 공식 문서에서 해당 XID 코드를 확인하세요." + + +@dataclass +class GpuInfo: + index: int = 0 + name: str = "N/A" + product_brand: str = "N/A" + product_architecture: str = "N/A" + driver_version: str = "N/A" + cuda_version: str = "N/A" + vbios_version: str = "N/A" + inforom_version: str = "N/A" + gpu_uuid: str = "N/A" + temperature: int = 0 + temperature_limit: str = "N/A" + power_draw: float = 0.0 + power_limit: float = 0.0 + power_state: str = "N/A" + memory_used: int = 0 + memory_total: int = 0 + memory_free: int = 0 + bar1_total: int = 0 + bar1_used: int = 0 + gpu_utilization: int = 0 + memory_utilization: int = 0 + ecc_mode: str = "N/A" + ecc_single_bit: int = 0 + ecc_double_bit: int = 0 + retired_pages_single: int = 0 + retired_pages_double: int = 0 + pci_address: str = "N/A" + pci_link_gen_current: str = "N/A" + pci_link_width_current: str = "N/A" + serial_number: str = "N/A" + fan_speed: int = 0 + persistence_mode: str = "N/A" + compute_mode: str = "N/A" + mig_mode: str = "N/A" + clock_graphics: str = "N/A" + clock_memory: str = "N/A" + clock_max_graphics: str = "N/A" + clock_max_memory: str = "N/A" + # --- HW Slowdown --- + hw_slowdown: str = "N/A" + hw_thermal_slowdown: str = "N/A" + hw_power_brake_slowdown: str = "N/A" + # --- RMA 관련 필드 --- + # Row Remapping + remap_corr_error: int = 0 + remap_uncorr_error: int = 0 + remap_pending: str = "N/A" + remap_failure: str = "N/A" + remap_bank_max: int = 0 + remap_bank_high: int = 0 + remap_bank_partial: int = 0 + remap_bank_low: int = 0 + remap_bank_none: int = 0 + # SRAM ECC + sram_corr_volatile: str = "N/A" + sram_uncorr_parity_volatile: str = "N/A" + sram_uncorr_secded_volatile: str = "N/A" + sram_corr_aggregate: str = "N/A" + sram_uncorr_parity_aggregate: str = "N/A" + sram_uncorr_secded_aggregate: str = "N/A" + sram_threshold_exceeded: str = "N/A" + # SRAM Sources + sram_l2: str = "N/A" + sram_sm: str = "N/A" + sram_microcontroller: str = "N/A" + sram_pcie: str = "N/A" + sram_other: str = "N/A" + # DRAM ECC + dram_corr_volatile: str = "N/A" + dram_uncorr_volatile: str = "N/A" + dram_corr_aggregate: str = "N/A" + dram_uncorr_aggregate: str = "N/A" + + +@dataclass +class RmaVerdict: + """GPU별 RMA 판정 결과""" + gpu_index: int = 0 + gpu_name: str = "" + serial_number: str = "" + qualifies_for_rma: bool = False + reasons: list = field(default_factory=list) + warnings: list = field(default_factory=list) + row_remap_details: dict = field(default_factory=dict) + sram_details: dict = field(default_factory=dict) + + +@dataclass +class KernelEvent: + timestamp: str + level: str + message: str + raw_line: str + + +@dataclass +class AnalysisResult: + filename: str = "" + file_size_mb: float = 0.0 + analysis_time: str = "" + gpus: list = field(default_factory=list) + driver_version: str = "N/A" + cuda_version: str = "N/A" + xid_events: list = field(default_factory=list) + xid_summary: dict = field(default_factory=dict) + kernel_events: list = field(default_factory=list) + kernel_summary: dict = field(default_factory=dict) + ecc_info: dict = field(default_factory=dict) + rma_verdicts: list = field(default_factory=list) # List[RmaVerdict] + rma_count: int = 0 # RMA 대상 GPU 수 + overall_status: str = "healthy" + total_issues: int = 0 + critical_count: int = 0 + warning_count: int = 0 + info_count: int = 0 + + +# --- 라인 단위 정규표현식 (성능 최적화) --- + +# XID: 한 라인 안에서만 매칭 +RE_XID = re.compile(r"NVRM:\s*Xid\s*\(PCI:([^)]+)\):\s*(\d+)") +RE_TIMESTAMP = re.compile(r"^(\w{3}\s+\d+\s+[\d:]+|\d{4}-\d{2}-\d{2}[T ][\d:]+|[\d]+\.[\d]+)") + +# GPU 정보 (라인 단위) - nvidia-smi -q 출력의 "Key : Value" 패턴 +RE_GPU_NAME = re.compile(r"Product Name\s*:\s*(.+)", re.IGNORECASE) +RE_GPU_BRAND = re.compile(r"Product Brand\s*:\s*(.+)", re.IGNORECASE) +RE_GPU_ARCH = re.compile(r"Product Architecture\s*:\s*(.+)", re.IGNORECASE) +RE_DRIVER = re.compile(r"Driver Version\s*:\s*([\d.]+)", re.IGNORECASE) +RE_CUDA = re.compile(r"CUDA Version\s*:\s*([\d.]+)", re.IGNORECASE) +RE_VBIOS = re.compile(r"VBIOS Version\s*:\s*(\S+)", re.IGNORECASE) +RE_INFOROM = re.compile(r"Image Version\s*:\s*(\S+)", re.IGNORECASE) +RE_UUID = re.compile(r"GPU UUID\s*:\s*(GPU-\S+)", re.IGNORECASE) +RE_TEMP = re.compile(r"GPU Current Temp\s*:\s*(\d+)\s*C", re.IGNORECASE) +RE_TEMP_LIMIT = re.compile(r"GPU T\.?Limit Temp\s*:\s*(.+)", re.IGNORECASE) +RE_POWER_DRAW = re.compile(r"Power Draw\s*:\s*([\d.]+)\s*W", re.IGNORECASE) +RE_POWER_LIMIT = re.compile(r"(?:Default |Enforced |Current )?Power Limit\s*:\s*([\d.]+)\s*W", re.IGNORECASE) +RE_POWER_STATE = re.compile(r"Power State\s*:\s*(\S+)", re.IGNORECASE) +RE_MEM_USED = re.compile(r"Used\s*:\s*(\d+)\s*MiB", re.IGNORECASE) +RE_MEM_TOTAL = re.compile(r"Total\s*:\s*(\d+)\s*MiB", re.IGNORECASE) +RE_MEM_FREE = re.compile(r"Free\s*:\s*(\d+)\s*MiB", re.IGNORECASE) +RE_GPU_UTIL = re.compile(r"Gpu\s*:\s*(\d+)\s*%", re.IGNORECASE) +RE_MEM_UTIL = re.compile(r"Memory\s*:\s*(\d+)\s*%", re.IGNORECASE) +RE_PCI = re.compile(r"Bus Id\s*:\s*([\dA-Fa-f:.]+)", re.IGNORECASE) +RE_PCI_GEN = re.compile(r"Current\s*:\s*Gen(\d+)", re.IGNORECASE) +RE_PCI_WIDTH = re.compile(r"Current\s*:\s*(\d+x)", re.IGNORECASE) +RE_SERIAL = re.compile(r"^\s*Serial Number\s*:\s*(\S+)", re.IGNORECASE) +RE_FAN = re.compile(r"Fan Speed\s*:\s*(\d+)\s*%", re.IGNORECASE) +RE_PERSIST = re.compile(r"Persistence Mode\s*:\s*(\w+)", re.IGNORECASE) +RE_COMPUTE_MODE = re.compile(r"Compute Mode\s*:\s*(.+)", re.IGNORECASE) +RE_MIG_MODE = re.compile(r"MIG Mode\s*:\s*(\w+)", re.IGNORECASE) +RE_ECC_MODE = re.compile(r"ECC Mode[\s\S]*?Current\s*:\s*(\w+)", re.IGNORECASE) +RE_CLOCK_GRAPHICS = re.compile(r"Graphics\s*:\s*(\d+)\s*MHz", re.IGNORECASE) +RE_CLOCK_MEMORY = re.compile(r"SM\s*:\s*(\d+)\s*MHz|Memory\s*:\s*(\d+)\s*MHz", re.IGNORECASE) +RE_HW_SLOWDOWN = re.compile(r"HW Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE) +RE_HW_THERMAL_SLOWDOWN = re.compile(r"HW Thermal Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE) +RE_HW_POWER_BRAKE_SLOWDOWN = re.compile(r"HW Power Brake Slowdown\s*:\s*(Active|Not Active)", re.IGNORECASE) +# GPU 섹션 헤더: "GPU 00000000:19:00.0" 또는 "GPU 0000:01:00.0" 또는 "GPU 0:" +RE_GPU_SECTION = re.compile(r"^GPU\s+([\dA-Fa-f]{4,8}:[\dA-Fa-f]{2}:[\dA-Fa-f]{2}\.\d)\s*$", re.IGNORECASE) +RE_GPU_SECTION_IDX = re.compile(r"^GPU\s+(\d+)\s*:\s*$", re.IGNORECASE) + +# nvidia-smi 테이블 행 +RE_SMI_ROW = re.compile( + r"\|\s*(\d+)\s+(.+?)\s+\w+\s*\|.*?(\d+)C.*?([\d.]+)W\s*/\s*([\d.]+)W.*?(\d+)MiB\s*/\s*(\d+)MiB.*?(\d+)%" +) + +# ECC (라인 단위) +RE_SINGLE_BIT = re.compile(r"Single Bit", re.IGNORECASE) +RE_DOUBLE_BIT = re.compile(r"Double Bit", re.IGNORECASE) +RE_AGGREGATE = re.compile(r"Aggregate\s*:\s*(\d+)", re.IGNORECASE) +RE_RETIRED = re.compile(r"Retired pages\s*:\s*(\d+)", re.IGNORECASE) +RE_PENDING = re.compile(r"Pending\s*.*:\s*Yes", re.IGNORECASE) + +# --- RMA 관련 정규표현식 --- +# Row Remapping +RE_REMAP_SECTION = re.compile(r"Remapped Rows", re.IGNORECASE) +RE_REMAP_CORR = re.compile(r"Correctable Error\s*:\s*(\d+)", re.IGNORECASE) +RE_REMAP_UNCORR = re.compile(r"Uncorrectable Error\s*:\s*(\d+)", re.IGNORECASE) +RE_REMAP_PENDING = re.compile(r"Pending\s*:\s*(Yes|No)", re.IGNORECASE) +RE_REMAP_FAILURE = re.compile(r"Remapping Failure Occurred\s*:\s*(Yes|No)", re.IGNORECASE) +RE_BANK_MAX = re.compile(r"Max\s*:\s*(\d+)\s*bank", re.IGNORECASE) +RE_BANK_HIGH = re.compile(r"High\s*:\s*(\d+)\s*bank", re.IGNORECASE) +RE_BANK_PARTIAL = re.compile(r"Partial\s*:\s*(\d+)\s*bank", re.IGNORECASE) +RE_BANK_LOW = re.compile(r"Low\s*:\s*(\d+)\s*bank", re.IGNORECASE) +RE_BANK_NONE = re.compile(r"None\s*:\s*(\d+)\s*bank", re.IGNORECASE) +# SRAM ECC +RE_SRAM_CORR = re.compile(r"SRAM Correctable\s*:\s*(\S+)", re.IGNORECASE) +RE_SRAM_UNCORR_PARITY = re.compile(r"SRAM Uncorrectable Parity\s*:\s*(\S+)", re.IGNORECASE) +RE_SRAM_UNCORR_SECDED = re.compile(r"SRAM Uncorrectable SEC-?DED\s*:\s*(\S+)", re.IGNORECASE) +RE_SRAM_THRESHOLD = re.compile(r"SRAM Threshold Exceeded\s*:\s*(\S+)", re.IGNORECASE) +# SRAM Sources +RE_SRAM_L2 = re.compile(r"SRAM L2\s*:\s*(\S+)", re.IGNORECASE) +RE_SRAM_SM = re.compile(r"SRAM SM\s*:\s*(\S+)", re.IGNORECASE) +RE_SRAM_MC = re.compile(r"SRAM Microcontroller\s*:\s*(\S+)", re.IGNORECASE) +RE_SRAM_PCIE = re.compile(r"SRAM PCIE\s*:\s*(\S+)", re.IGNORECASE) +RE_SRAM_OTHER = re.compile(r"SRAM Other\s*:\s*(\S+)", re.IGNORECASE) +# DRAM ECC +RE_DRAM_CORR = re.compile(r"DRAM Correctable\s*:\s*(\S+)", re.IGNORECASE) +RE_DRAM_UNCORR = re.compile(r"DRAM Uncorrectable\s*:\s*(\S+)", re.IGNORECASE) +# ECC Section context +RE_ECC_SECTION = re.compile(r"ECC Errors", re.IGNORECASE) +RE_VOLATILE = re.compile(r"^\s*Volatile\s*$", re.IGNORECASE) +RE_AGGREGATE_SECTION = re.compile(r"^\s*Aggregate\s*$", re.IGNORECASE) +RE_AGG_SRAM_SOURCES = re.compile(r"Aggregate Uncorrectable SRAM Sources", re.IGNORECASE) + +# 커널 이벤트 (라인 단위 키워드 매칭) +KERNEL_CRITICAL_KW = re.compile( + r"(kernel panic|hardware error|machine check|fatal|BUG:|RIP:)", re.IGNORECASE +) +KERNEL_ERROR_KW = re.compile( + r"(error|failed|failure|segfault|cannot|unable to)", re.IGNORECASE +) +KERNEL_OOM_KW = re.compile( + r"(Out of memory|oom-killer|invoked oom|Killed process)", re.IGNORECASE +) +KERNEL_PCIE_KW = re.compile( + r"(AER|correctable error|uncorrectable error)", re.IGNORECASE +) +KERNEL_SKIP_KW = re.compile(r"(audit|systemd|session|polkit|dbus)", re.IGNORECASE) + +# 섹션 구분: 언더스코어(_) 또는 대시(-)가 양옆에 있고 대괄호([, ])가 있을 수도 없을 수도 있는 구분선 매칭 +# nvidia-bug-report.log 내 dmesg 등 다양한 로그 섹션 구분선을 감지합니다. +RE_SECTION = re.compile(r"^[_-]+\s*\[?(.+?)\]?\s*[_-]+$") + + +# --- 파서 함수들 --- + +_RE_PCI_NORMALIZE = re.compile( + r"^([\dA-Fa-f]+):([\dA-Fa-f]{1,2}):([\dA-Fa-f]{1,2})(?:\.\d)?$" +) +_RE_PCI_NO_DOMAIN = re.compile( + r"^([\dA-Fa-f]{1,2}):([\dA-Fa-f]{1,2})(?:\.\d)?$" +) + +def _norm_pci(addr: str) -> str: + """ + PCI 주소를 'DOMAIN:BUS:DEVICE' 형태로 정규화. + 도메인은 정수값으로, bus/device는 소문자 2자리 hex로 통일. + - '0000:19:00' → '0:19:00' + - '00000000:19:00.0' → '0:19:00' + - '0001:3b:00.0' → '1:3b:00' (다른 도메인 → 다른 키) + - '19:00.0' → '0:19:00' (도메인 생략 → 0으로 가정) + 매칭 실패 시 원문 소문자 반환. + """ + s = addr.strip() + m = _RE_PCI_NORMALIZE.match(s) + if m: + domain = int(m.group(1), 16) + bus = m.group(2).zfill(2).lower() + device = m.group(3).zfill(2).lower() + return f"{domain}:{bus}:{device}" + m2 = _RE_PCI_NO_DOMAIN.match(s) + if m2: + bus = m2.group(1).zfill(2).lower() + device = m2.group(2).zfill(2).lower() + return f"0:{bus}:{device}" + return s.lower() + + +def read_log_file(file_content: bytes, filename: str = "") -> str: + """gz 또는 텍스트 파일을 읽어 문자열로 반환""" + if filename.endswith(".gz") or file_content[:2] == b'\x1f\x8b': + try: + return gzip.decompress(file_content).decode("utf-8", errors="replace") + except gzip.BadGzipFile: + return file_content.decode("utf-8", errors="replace") + return file_content.decode("utf-8", errors="replace") + + +def analyze_log(file_content: bytes, filename: str = "uploaded_file") -> AnalysisResult: + """메인 분석 함수 - 라인 단위 싱글 패스 처리""" + result = AnalysisResult( + filename=filename, + file_size_mb=round(len(file_content) / (1024 * 1024), 2), + analysis_time=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + ) + + log_text = read_log_file(file_content, filename) + lines = log_text.splitlines() + + # --- 상태 변수들 --- + current_section = "header" + driver_version = None + cuda_version = None + + # GPU 파싱 상태 + gpu_list = [] + current_gpu = None + smi_gpus_found = False + + # NVSMI LOG 상세 섹션의 PCI → Serial 별도 수집 (smi_gpus_found 여부 무관) + pci_serial_map: dict = {} # { "00000000:19:00.0" : "serial_string" } + _nvsmi_cur_pci: str = "" # 현재 GPU PCI (NVSMI 상세 파싱용) + + # XID 이벤트 + xid_events = [] + + # 커널 이벤트 (dmesg 섹션만) + kernel_events = [] + seen_kernel = set() + in_dmesg = False + + # RMA/ECC 파싱 상태 머신 + in_ecc_section = False + ecc_subsection = None # "volatile", "aggregate", "agg_sources" + in_remap_section = False + in_bank_histogram = False + + # ECC 상태 + ecc_single_total = 0 + ecc_double_total = 0 + ecc_retired = 0 + ecc_pending = False + ecc_context = None # "single" or "double" 마지막으로 본 ECC 유형 + + # --- 라인 단위 파싱 (싱글 패스) --- + for line in lines: + stripped = line.strip() + if not stripped: + continue + + # 섹션 감지 + sec_match = RE_SECTION.match(stripped) + if sec_match: + current_section = sec_match.group(1).strip().lower() + in_dmesg = any(kw in current_section for kw in ["dmesg", "kernel", "messages"]) + continue + + # --- NVSMI LOG: GPU PCI 헤더 및 Serial Number 항상 수집 --- + gpu_sec_always = RE_GPU_SECTION.match(stripped) + if gpu_sec_always: + _nvsmi_cur_pci = _norm_pci(gpu_sec_always.group(1)) + elif _nvsmi_cur_pci: + serial_m = RE_SERIAL.search(stripped) + if serial_m and "chassis" not in stripped.lower(): + val = serial_m.group(1).strip() + if val and val != "N/A": + pci_serial_map[_nvsmi_cur_pci] = val + + # --- XID 에러 (모든 섹션) --- + xid_m = RE_XID.search(stripped) + if xid_m: + ts_m = RE_TIMESTAMP.match(stripped) + timestamp = ts_m.group(1) if ts_m else "" + xid_events.append(XidEvent( + timestamp=timestamp, + xid_code=int(xid_m.group(2)), + pci_address=xid_m.group(1).strip(), + raw_line=stripped[:500], + )) + continue + + # --- 드라이버/CUDA 버전 (최초 발견만) --- + if not driver_version: + dm = RE_DRIVER.search(stripped) + if dm: + driver_version = dm.group(1) + if not cuda_version: + cm = RE_CUDA.search(stripped) + if cm: + cuda_version = cm.group(1) + + # --- nvidia-smi 테이블 행 --- + smi_m = RE_SMI_ROW.search(stripped) + if smi_m: + smi_gpus_found = True + gpu_list.append(GpuInfo( + index=int(smi_m.group(1)), + name=smi_m.group(2).strip(), + temperature=int(smi_m.group(3)), + power_draw=float(smi_m.group(4)), + power_limit=float(smi_m.group(5)), + memory_used=int(smi_m.group(6)), + memory_total=int(smi_m.group(7)), + gpu_utilization=int(smi_m.group(8)), + )) + continue + + # --- nvidia-smi -q 상세 형식 (GPU 섹션 기반) --- + if not smi_gpus_found: + # GPU 섹션 헤더 감지: "GPU 00000000:19:00.0" 또는 "GPU 0:" + gpu_sec_m = RE_GPU_SECTION.match(stripped) or RE_GPU_SECTION_IDX.match(stripped) + if gpu_sec_m: + if current_gpu and current_gpu.name != "N/A": + gpu_list.append(current_gpu) + current_gpu = GpuInfo(index=len(gpu_list)) + # PCI 주소가 헤더에 포함된 경우 + addr = gpu_sec_m.group(1) + if ":" in addr and "." in addr: + current_gpu.pci_address = addr + continue + + if current_gpu is not None: + # 제품 정보 + m = RE_GPU_NAME.search(stripped) + if m: + current_gpu.name = m.group(1).strip() + continue + m = RE_GPU_BRAND.search(stripped) + if m: + current_gpu.product_brand = m.group(1).strip() + continue + m = RE_GPU_ARCH.search(stripped) + if m: + current_gpu.product_architecture = m.group(1).strip() + continue + m = RE_UUID.search(stripped) + if m: + current_gpu.gpu_uuid = m.group(1).strip() + continue + m = RE_SERIAL.search(stripped) + if m and "chassis" not in stripped.lower(): + val = m.group(1).strip() + if val != "N/A": + current_gpu.serial_number = val + continue + m = RE_VBIOS.search(stripped) + if m: + current_gpu.vbios_version = m.group(1).strip() + continue + m = RE_INFOROM.search(stripped) + if m: + current_gpu.inforom_version = m.group(1).strip() + continue + + # 온도 + m = RE_TEMP.search(stripped) + if m: + current_gpu.temperature = int(m.group(1)) + continue + m = RE_TEMP_LIMIT.search(stripped) + if m: + current_gpu.temperature_limit = m.group(1).strip() + continue + + # 전력 + m = RE_POWER_DRAW.search(stripped) + if m: + current_gpu.power_draw = float(m.group(1)) + continue + m = RE_POWER_LIMIT.search(stripped) + if m: + current_gpu.power_limit = float(m.group(1)) + continue + m = RE_POWER_STATE.search(stripped) + if m: + current_gpu.power_state = m.group(1).strip() + continue + + # 메모리 (FB Memory Usage 값만 유지, 이후 Conf Compute 등에 덮어쓰기 방지) + m = RE_MEM_TOTAL.search(stripped) + if m: + val = int(m.group(1)) + if current_gpu.memory_total == 0 and val > 0: + current_gpu.memory_total = val + continue + m = RE_MEM_USED.search(stripped) + if m: + val = int(m.group(1)) + if current_gpu.memory_used == 0: + current_gpu.memory_used = val + continue + m = RE_MEM_FREE.search(stripped) + if m: + val = int(m.group(1)) + if current_gpu.memory_free == 0 and val > 0: + current_gpu.memory_free = val + continue + + # 사용률 + m = RE_GPU_UTIL.search(stripped) + if m: + current_gpu.gpu_utilization = int(m.group(1)) + continue + m = RE_MEM_UTIL.search(stripped) + if m: + current_gpu.memory_utilization = int(m.group(1)) + continue + + # PCI + m = RE_PCI.search(stripped) + if m: + current_gpu.pci_address = m.group(1).strip() + continue + m = RE_PCI_GEN.search(stripped) + if m: + current_gpu.pci_link_gen_current = f"Gen{m.group(1)}" + continue + m = RE_PCI_WIDTH.search(stripped) + if m: + current_gpu.pci_link_width_current = m.group(1) + continue + + # 기타 + m = RE_FAN.search(stripped) + if m: + current_gpu.fan_speed = int(m.group(1)) + continue + m = RE_PERSIST.search(stripped) + if m: + current_gpu.persistence_mode = m.group(1).strip() + continue + m = RE_COMPUTE_MODE.search(stripped) + if m: + current_gpu.compute_mode = m.group(1).strip() + continue + m = RE_MIG_MODE.search(stripped) + if m: + current_gpu.mig_mode = m.group(1).strip() + continue + m = RE_ECC_MODE.search(stripped) + if m: + current_gpu.ecc_mode = m.group(1).strip() + continue + m = RE_CLOCK_GRAPHICS.search(stripped) + if m: + current_gpu.clock_graphics = f"{m.group(1)} MHz" + continue + + # HW Slowdown (순서 중요: 더 구체적인 패턴 먼저) + m = RE_HW_THERMAL_SLOWDOWN.search(stripped) + if m: + current_gpu.hw_thermal_slowdown = m.group(1) + continue + m = RE_HW_POWER_BRAKE_SLOWDOWN.search(stripped) + if m: + current_gpu.hw_power_brake_slowdown = m.group(1) + continue + m = RE_HW_SLOWDOWN.search(stripped) + if m: + current_gpu.hw_slowdown = m.group(1) + continue + + # --- RMA: ECC 섹션 파싱 (GPU 내부) --- + if RE_ECC_SECTION.match(stripped): + in_ecc_section = True + ecc_subsection = None + continue + if in_ecc_section: + if RE_VOLATILE.match(stripped): + ecc_subsection = "volatile" + continue + if RE_AGGREGATE_SECTION.match(stripped): + ecc_subsection = "aggregate" + continue + if RE_AGG_SRAM_SOURCES.search(stripped): + ecc_subsection = "agg_sources" + continue + + if ecc_subsection == "volatile": + m = RE_SRAM_CORR.search(stripped) + if m: current_gpu.sram_corr_volatile = m.group(1); continue + m = RE_SRAM_UNCORR_PARITY.search(stripped) + if m: current_gpu.sram_uncorr_parity_volatile = m.group(1); continue + m = RE_SRAM_UNCORR_SECDED.search(stripped) + if m: current_gpu.sram_uncorr_secded_volatile = m.group(1); continue + m = RE_DRAM_CORR.search(stripped) + if m: current_gpu.dram_corr_volatile = m.group(1); continue + m = RE_DRAM_UNCORR.search(stripped) + if m: current_gpu.dram_uncorr_volatile = m.group(1); continue + + elif ecc_subsection == "aggregate": + m = RE_SRAM_CORR.search(stripped) + if m: current_gpu.sram_corr_aggregate = m.group(1); continue + m = RE_SRAM_UNCORR_PARITY.search(stripped) + if m: current_gpu.sram_uncorr_parity_aggregate = m.group(1); continue + m = RE_SRAM_UNCORR_SECDED.search(stripped) + if m: current_gpu.sram_uncorr_secded_aggregate = m.group(1); continue + m = RE_DRAM_CORR.search(stripped) + if m: current_gpu.dram_corr_aggregate = m.group(1); continue + m = RE_DRAM_UNCORR.search(stripped) + if m: current_gpu.dram_uncorr_aggregate = m.group(1); continue + m = RE_SRAM_THRESHOLD.search(stripped) + if m: current_gpu.sram_threshold_exceeded = m.group(1); continue + + elif ecc_subsection == "agg_sources": + m = RE_SRAM_L2.search(stripped) + if m: current_gpu.sram_l2 = m.group(1); continue + m = RE_SRAM_SM.search(stripped) + if m: current_gpu.sram_sm = m.group(1); continue + m = RE_SRAM_MC.search(stripped) + if m: current_gpu.sram_microcontroller = m.group(1); continue + m = RE_SRAM_PCIE.search(stripped) + if m: current_gpu.sram_pcie = m.group(1); continue + m = RE_SRAM_OTHER.search(stripped) + if m: current_gpu.sram_other = m.group(1); continue + + # --- RMA: Remapped Rows 섹션 파싱 (GPU 내부) --- + if RE_REMAP_SECTION.match(stripped): + in_remap_section = True + in_ecc_section = False + in_bank_histogram = False + continue + if in_remap_section: + m = RE_REMAP_CORR.search(stripped) + if m and "remap" not in stripped.lower(): + current_gpu.remap_corr_error = int(m.group(1)); continue + m = RE_REMAP_UNCORR.search(stripped) + if m: + current_gpu.remap_uncorr_error = int(m.group(1)); continue + m = RE_REMAP_FAILURE.search(stripped) + if m: + current_gpu.remap_failure = m.group(1); continue + m = RE_REMAP_PENDING.search(stripped) + if m and "page" not in stripped.lower(): + current_gpu.remap_pending = m.group(1); continue + if "Bank Remap Availability" in stripped: + in_bank_histogram = True; continue + if in_bank_histogram: + m = RE_BANK_MAX.search(stripped) + if m: current_gpu.remap_bank_max = int(m.group(1)); continue + m = RE_BANK_HIGH.search(stripped) + if m: current_gpu.remap_bank_high = int(m.group(1)); continue + m = RE_BANK_PARTIAL.search(stripped) + if m: current_gpu.remap_bank_partial = int(m.group(1)); continue + m = RE_BANK_LOW.search(stripped) + if m: current_gpu.remap_bank_low = int(m.group(1)); continue + m = RE_BANK_NONE.search(stripped) + if m: + current_gpu.remap_bank_none = int(m.group(1)) + in_remap_section = False + in_bank_histogram = False + continue + + # --- ECC 에러 전체 집계 (라인 단위 상태 머신) --- + if RE_SINGLE_BIT.search(stripped) and "sram" not in stripped.lower(): + ecc_context = "single" + continue + if RE_DOUBLE_BIT.search(stripped) and "sram" not in stripped.lower(): + ecc_context = "double" + continue + if ecc_context: + agg_m = RE_AGGREGATE.search(stripped) + if agg_m: + val = int(agg_m.group(1)) + if ecc_context == "single": + ecc_single_total = max(ecc_single_total, val) + else: + ecc_double_total = max(ecc_double_total, val) + ecc_context = None + continue + + ret_m = RE_RETIRED.search(stripped) + if ret_m: + ecc_retired = max(ecc_retired, int(ret_m.group(1))) + continue + if RE_PENDING.search(stripped) and "remap" not in stripped.lower(): + ecc_pending = True + continue + + # --- 커널 이벤트 (dmesg 섹션) --- + if in_dmesg and stripped not in seen_kernel: + if KERNEL_SKIP_KW.search(stripped): + continue + + level = None + keyword = "" + # 우선순위: oom > critical > pcie > error + m = KERNEL_OOM_KW.search(stripped) + if m: + level, keyword = "oom", m.group(1) + if not level: + m = KERNEL_CRITICAL_KW.search(stripped) + if m: + level, keyword = "critical", m.group(1) + if not level: + m = KERNEL_PCIE_KW.search(stripped) + if m: + level, keyword = "pcie_error", m.group(1) + if not level: + m = KERNEL_ERROR_KW.search(stripped) + if m: + level, keyword = "error", m.group(1) + + if level: + seen_kernel.add(stripped) + ts_m = RE_TIMESTAMP.match(stripped) + kernel_events.append(KernelEvent( + timestamp=ts_m.group(1) if ts_m else "", + level=level, + message=keyword, + raw_line=stripped[:500], + )) + + # --- 마지막 GPU 저장 --- + if current_gpu and current_gpu.name != "N/A" and not smi_gpus_found: + gpu_list.append(current_gpu) + + # --- 결과 조립 --- + result.driver_version = driver_version or "N/A" + result.cuda_version = cuda_version or "N/A" + + for gpu in gpu_list: + gpu.driver_version = result.driver_version + gpu.cuda_version = result.cuda_version + result.gpus = gpu_list + + # gpu_list 시리얼 보완: NVSMI LOG에서 수집한 pci_serial_map 활용 + for gpu in gpu_list: + if (not gpu.serial_number or gpu.serial_number == "N/A") and gpu.pci_address: + s = pci_serial_map.get(_norm_pci(gpu.pci_address)) + if s: + gpu.serial_number = s + + # PCI 주소 → GPU 매핑 (정규화 키 사용) + pci_to_gpu = {} + for gpu in gpu_list: + if gpu.pci_address and gpu.pci_address != "N/A": + pci_to_gpu[_norm_pci(gpu.pci_address)] = gpu + for event in xid_events: + key = _norm_pci(event.pci_address) + matched = pci_to_gpu.get(key) + if matched: + event.gpu_index = matched.index + event.gpu_name = matched.name + event.gpu_serial = matched.serial_number or pci_serial_map.get(key, "") + else: + # gpu_list에 없어도 pci_serial_map에서 시리얼 보완 + event.gpu_serial = pci_serial_map.get(key, "") + + result.xid_events = xid_events + xid_counter = Counter(e.xid_code for e in xid_events) + result.xid_summary = { + code: { + "count": count, + "severity": XID_DB["xid_errors"].get(str(code), {}).get("severity", "warning"), + "name": XID_DB["xid_errors"].get(str(code), {}).get("name", f"Unknown XID {code}"), + "gpus": sorted({ + f"GPU {e.gpu_index} {e.gpu_name} [S/N: {e.gpu_serial}]" if e.gpu_serial + else (f"GPU {e.gpu_index} ({e.gpu_name})" if e.gpu_name else e.pci_address) + for e in xid_events if e.xid_code == code + }), + } + for code, count in xid_counter.most_common() + } + + result.kernel_events = kernel_events + kernel_counter = Counter(e.level for e in kernel_events) + result.kernel_summary = dict(kernel_counter) + + result.ecc_info = { + "single_bit_total": ecc_single_total, + "double_bit_total": ecc_double_total, + "retired_pages": ecc_retired, + "pending_retirement": ecc_pending, + } + + # --- RMA 판정 --- + result.rma_verdicts = [evaluate_rma(gpu) for gpu in gpu_list] + result.rma_count = sum(1 for v in result.rma_verdicts if v.qualifies_for_rma) + + # 상태 판정 + critical_xids = sum(1 for e in xid_events if e.severity == "critical") + warning_xids = sum(1 for e in xid_events if e.severity == "warning") + + result.critical_count = ( + critical_xids + + kernel_counter.get("critical", 0) + + kernel_counter.get("oom", 0) + + (1 if ecc_double_total > 0 else 0) + + result.rma_count # RMA 대상 GPU 수도 critical에 포함 + ) + result.warning_count = ( + warning_xids + + kernel_counter.get("error", 0) + + kernel_counter.get("pcie_error", 0) + + (1 if ecc_single_total > 10 else 0) + ) + result.info_count = sum(1 for e in xid_events if e.severity == "info") + result.total_issues = result.critical_count + result.warning_count + + if result.rma_count > 0: + result.overall_status = "critical" + elif result.critical_count > 0: + result.overall_status = "critical" + elif result.warning_count > 0: + result.overall_status = "warning" + else: + result.overall_status = "healthy" + + return result + + +def _safe_int(val: str) -> int: + """N/A 등 숫자가 아닌 값을 0으로 변환""" + try: + return int(val) + except (ValueError, TypeError): + return 0 + + +def evaluate_rma(gpu: GpuInfo) -> RmaVerdict: + """ + NVIDIA Hopper RMA 기준에 따라 GPU의 RMA 적격 여부를 판정합니다. + + [Row-Remapping RMA 기준] + 1. Remapping Failure Occurred = Yes → 즉시 RMA + 2. 한 Bank에서 Uncorrectable Error Row 8회 이상 리매핑 + 3. 총 512회 리매핑 발생 + 4. 리매핑 8회 후 Field Diagnostics에서 문제 감지 + + [SRAM RMA 기준] + 1. SRAM Threshold Exceeded = Yes → 즉시 RMA + 2. Parity SRAM: 주소 뱅크 내 4개 이상 UCE Unique Count + 3. SECDED ECC SRAM: 주소 뱅크 내 2개 이상 UCE Unique Count + """ + verdict = RmaVerdict( + gpu_index=gpu.index, + gpu_name=gpu.name, + serial_number=gpu.serial_number, + ) + + # --- Row-Remapping 분석 --- + total_remaps = gpu.remap_corr_error + gpu.remap_uncorr_error + + verdict.row_remap_details = { + "correctable_error": gpu.remap_corr_error, + "uncorrectable_error": gpu.remap_uncorr_error, + "total_remaps": total_remaps, + "pending": gpu.remap_pending, + "failure_occurred": gpu.remap_failure, + "bank_max": gpu.remap_bank_max, + "bank_high": gpu.remap_bank_high, + "bank_partial": gpu.remap_bank_partial, + "bank_low": gpu.remap_bank_low, + "bank_none": gpu.remap_bank_none, + } + + # RMA 기준 1: Remapping Failure Occurred = Yes + if gpu.remap_failure.lower() == "yes": + verdict.qualifies_for_rma = True + verdict.reasons.append( + "Remapping Failure Occurred = Yes: " + "리매핑 실패 플래그 설정됨. 즉시 RMA 대상." + ) + + # RMA 기준 2: Uncorrectable Error Row 8회 이상 (Bank당) + if gpu.remap_uncorr_error >= 8: + verdict.qualifies_for_rma = True + verdict.reasons.append( + f"Uncorrectable Error 리매핑 {gpu.remap_uncorr_error}회: " + f"Bank당 8회 이상 Uncorrectable Error 리매핑 발생 (기준: 8회)." + ) + + # RMA 기준 3: 총 512회 리매핑 + if total_remaps >= 512: + verdict.qualifies_for_rma = True + verdict.reasons.append( + f"총 리매핑 {total_remaps}회: " + f"Uncorrectable Memory Error로 총 512회 이상 리매핑 발생 (기준: 512회)." + ) + + # 경고: 리매핑 진행 중 + if gpu.remap_pending.lower() == "yes": + verdict.warnings.append( + "Row Remapping Pending = Yes: 리매핑 대기 중. 재부팅 후 적용됩니다." + ) + + # 경고: Bank 가용성 낮음 + if gpu.remap_bank_none > 0: + verdict.warnings.append( + f"Bank Remap None = {gpu.remap_bank_none}: " + f"리매핑 가용 뱅크가 소진된 뱅크가 있습니다." + ) + if gpu.remap_bank_low > 0: + verdict.warnings.append( + f"Bank Remap Low = {gpu.remap_bank_low}: " + f"리매핑 가용 뱅크가 적은 뱅크가 있습니다." + ) + + # 경고: 리매핑이 상당수 발생 + if 0 < gpu.remap_uncorr_error < 8: + verdict.warnings.append( + f"Uncorrectable Error 리매핑 {gpu.remap_uncorr_error}회: " + f"아직 RMA 기준(8회) 미달이나 모니터링 필요." + ) + + # --- SRAM 분석 --- + sram_uncorr_parity = _safe_int(gpu.sram_uncorr_parity_aggregate) + sram_uncorr_secded = _safe_int(gpu.sram_uncorr_secded_aggregate) + + verdict.sram_details = { + "volatile": { + "correctable": gpu.sram_corr_volatile, + "uncorrectable_parity": gpu.sram_uncorr_parity_volatile, + "uncorrectable_secded": gpu.sram_uncorr_secded_volatile, + }, + "aggregate": { + "correctable": gpu.sram_corr_aggregate, + "uncorrectable_parity": gpu.sram_uncorr_parity_aggregate, + "uncorrectable_secded": gpu.sram_uncorr_secded_aggregate, + "threshold_exceeded": gpu.sram_threshold_exceeded, + }, + "sources": { + "l2": gpu.sram_l2, + "sm": gpu.sram_sm, + "microcontroller": gpu.sram_microcontroller, + "pcie": gpu.sram_pcie, + "other": gpu.sram_other, + }, + "dram": { + "corr_volatile": gpu.dram_corr_volatile, + "uncorr_volatile": gpu.dram_uncorr_volatile, + "corr_aggregate": gpu.dram_corr_aggregate, + "uncorr_aggregate": gpu.dram_uncorr_aggregate, + }, + } + + # SRAM RMA 기준 1: Threshold Exceeded = Yes + if gpu.sram_threshold_exceeded.lower() == "yes": + verdict.qualifies_for_rma = True + verdict.reasons.append( + "SRAM Threshold Exceeded = Yes: " + "SRAM UCE 임계치 초과 플래그 설정됨. 즉시 RMA 대상." + ) + + # SRAM RMA 기준 2: Parity SRAM UCE >= 4 + if sram_uncorr_parity >= 4: + verdict.qualifies_for_rma = True + verdict.reasons.append( + f"SRAM Uncorrectable Parity (Aggregate) = {sram_uncorr_parity}: " + f"Parity 보호 SRAM에서 4개 이상 UCE 발생 (기준: 4개)." + ) + + # SRAM RMA 기준 3: SECDED ECC SRAM UCE >= 2 + if sram_uncorr_secded >= 2: + verdict.qualifies_for_rma = True + verdict.reasons.append( + f"SRAM Uncorrectable SEC-DED (Aggregate) = {sram_uncorr_secded}: " + f"SECDED ECC 보호 SRAM에서 2개 이상 UCE 발생 (기준: 2개)." + ) + + # 경고: SRAM UCE가 있지만 기준 미달 + if 0 < sram_uncorr_parity < 4: + verdict.warnings.append( + f"SRAM Parity UCE = {sram_uncorr_parity}: " + f"아직 RMA 기준(4개) 미달이나 모니터링 필요." + ) + if 0 < sram_uncorr_secded < 2: + verdict.warnings.append( + f"SRAM SECDED UCE = {sram_uncorr_secded}: " + f"아직 RMA 기준(2개) 미달이나 모니터링 필요." + ) + + return verdict diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5051e7a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +streamlit>=1.30.0 +pandas>=2.0.0