feat: merge Xid-Catalog.xlsx, fix dmesg regex, and improve history UI
This commit is contained in:
unknown
@@ -0,0 +1,412 @@
|
||||
"""
|
||||
NV-Log Analyzer - SQLite 데이터베이스 모듈
|
||||
분석 결과를 영구 저장하여 장애 케이스 이력을 축적합니다.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
DB_PATH = Path(__file__).parent / "data" / "nv_log_history.db"
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_conn():
|
||||
"""DB 커넥션 컨텍스트 매니저"""
|
||||
conn = sqlite3.connect(str(DB_PATH))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
try:
|
||||
yield conn
|
||||
conn.commit()
|
||||
except Exception:
|
||||
conn.rollback()
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def init_db():
|
||||
"""DB 테이블 초기화 (앱 시작 시 1회 호출)"""
|
||||
with get_conn() as conn:
|
||||
conn.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS analysis (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now','localtime')),
|
||||
filename TEXT NOT NULL,
|
||||
file_size_mb REAL,
|
||||
overall_status TEXT,
|
||||
total_issues INTEGER DEFAULT 0,
|
||||
critical_count INTEGER DEFAULT 0,
|
||||
warning_count INTEGER DEFAULT 0,
|
||||
info_count INTEGER DEFAULT 0,
|
||||
driver_version TEXT,
|
||||
cuda_version TEXT,
|
||||
gpu_count INTEGER DEFAULT 0,
|
||||
xid_count INTEGER DEFAULT 0,
|
||||
kernel_event_count INTEGER DEFAULT 0,
|
||||
rma_count INTEGER DEFAULT 0,
|
||||
ecc_info TEXT,
|
||||
memo TEXT DEFAULT ''
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS analysis_gpu (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
analysis_id INTEGER NOT NULL REFERENCES analysis(id) ON DELETE CASCADE,
|
||||
gpu_index INTEGER,
|
||||
name TEXT,
|
||||
serial_number TEXT,
|
||||
gpu_uuid TEXT,
|
||||
pci_address TEXT,
|
||||
product_brand TEXT,
|
||||
product_architecture TEXT,
|
||||
driver_version TEXT,
|
||||
cuda_version TEXT,
|
||||
vbios_version TEXT,
|
||||
inforom_version TEXT,
|
||||
temperature INTEGER,
|
||||
temperature_limit TEXT,
|
||||
power_draw REAL,
|
||||
power_limit REAL,
|
||||
power_state TEXT,
|
||||
memory_used INTEGER,
|
||||
memory_total INTEGER,
|
||||
memory_free INTEGER,
|
||||
gpu_utilization INTEGER,
|
||||
memory_utilization INTEGER,
|
||||
fan_speed INTEGER,
|
||||
persistence_mode TEXT,
|
||||
compute_mode TEXT,
|
||||
mig_mode TEXT,
|
||||
ecc_mode TEXT,
|
||||
pci_link_gen TEXT,
|
||||
pci_link_width TEXT,
|
||||
clock_graphics TEXT,
|
||||
clock_memory TEXT,
|
||||
-- RMA 관련
|
||||
remap_corr_error INTEGER DEFAULT 0,
|
||||
remap_uncorr_error INTEGER DEFAULT 0,
|
||||
remap_pending TEXT,
|
||||
remap_failure TEXT,
|
||||
remap_bank_max INTEGER DEFAULT 0,
|
||||
remap_bank_high INTEGER DEFAULT 0,
|
||||
remap_bank_partial INTEGER DEFAULT 0,
|
||||
remap_bank_low INTEGER DEFAULT 0,
|
||||
remap_bank_none INTEGER DEFAULT 0,
|
||||
sram_corr_volatile TEXT,
|
||||
sram_uncorr_parity_volatile TEXT,
|
||||
sram_uncorr_secded_volatile TEXT,
|
||||
sram_corr_aggregate TEXT,
|
||||
sram_uncorr_parity_aggregate TEXT,
|
||||
sram_uncorr_secded_aggregate TEXT,
|
||||
sram_threshold_exceeded TEXT,
|
||||
sram_l2 TEXT,
|
||||
sram_sm TEXT,
|
||||
sram_microcontroller TEXT,
|
||||
sram_pcie TEXT,
|
||||
sram_other TEXT,
|
||||
dram_corr_volatile TEXT,
|
||||
dram_uncorr_volatile TEXT,
|
||||
dram_corr_aggregate TEXT,
|
||||
dram_uncorr_aggregate TEXT,
|
||||
-- HW Slowdown
|
||||
hw_slowdown TEXT,
|
||||
hw_thermal_slowdown TEXT,
|
||||
hw_power_brake_slowdown TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS analysis_rma (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
analysis_id INTEGER NOT NULL REFERENCES analysis(id) ON DELETE CASCADE,
|
||||
gpu_index INTEGER,
|
||||
gpu_name TEXT,
|
||||
serial_number TEXT,
|
||||
qualifies_for_rma INTEGER DEFAULT 0,
|
||||
reasons TEXT,
|
||||
warnings TEXT,
|
||||
row_remap_details TEXT,
|
||||
sram_details TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS analysis_xid (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
analysis_id INTEGER NOT NULL REFERENCES analysis(id) ON DELETE CASCADE,
|
||||
timestamp TEXT,
|
||||
xid_code INTEGER,
|
||||
xid_name TEXT,
|
||||
severity TEXT,
|
||||
pci_address TEXT,
|
||||
gpu_name TEXT,
|
||||
gpu_serial TEXT,
|
||||
description TEXT,
|
||||
action TEXT,
|
||||
raw_line TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS analysis_kernel (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
analysis_id INTEGER NOT NULL REFERENCES analysis(id) ON DELETE CASCADE,
|
||||
timestamp TEXT,
|
||||
level TEXT,
|
||||
message TEXT,
|
||||
raw_line TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_analysis_created ON analysis(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_analysis_status ON analysis(overall_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_xid_code ON analysis_xid(xid_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_xid_severity ON analysis_xid(severity);
|
||||
CREATE INDEX IF NOT EXISTS idx_gpu_serial ON analysis_gpu(serial_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_kernel_level ON analysis_kernel(level);
|
||||
CREATE INDEX IF NOT EXISTS idx_rma_serial ON analysis_rma(serial_number);
|
||||
CREATE INDEX IF NOT EXISTS idx_rma_qualifies ON analysis_rma(qualifies_for_rma);
|
||||
""")
|
||||
# 기존 DB 마이그레이션: gpu_name 컬럼이 없으면 추가
|
||||
cols = {row[1] for row in conn.execute("PRAGMA table_info(analysis_xid)")}
|
||||
if "gpu_name" not in cols:
|
||||
conn.execute("ALTER TABLE analysis_xid ADD COLUMN gpu_name TEXT")
|
||||
if "gpu_serial" not in cols:
|
||||
conn.execute("ALTER TABLE analysis_xid ADD COLUMN gpu_serial TEXT")
|
||||
# HW slowdown 마이그레이션
|
||||
gpu_cols = {row[1] for row in conn.execute("PRAGMA table_info(analysis_gpu)")}
|
||||
if "hw_slowdown" not in gpu_cols:
|
||||
conn.execute("ALTER TABLE analysis_gpu ADD COLUMN hw_slowdown TEXT")
|
||||
if "hw_thermal_slowdown" not in gpu_cols:
|
||||
conn.execute("ALTER TABLE analysis_gpu ADD COLUMN hw_thermal_slowdown TEXT")
|
||||
if "hw_power_brake_slowdown" not in gpu_cols:
|
||||
conn.execute("ALTER TABLE analysis_gpu ADD COLUMN hw_power_brake_slowdown TEXT")
|
||||
|
||||
|
||||
def save_analysis(result) -> int:
|
||||
"""분석 결과 전체를 DB에 저장하고 analysis_id 반환"""
|
||||
with get_conn() as conn:
|
||||
cur = conn.execute("""
|
||||
INSERT INTO analysis
|
||||
(filename, file_size_mb, overall_status,
|
||||
total_issues, critical_count, warning_count, info_count,
|
||||
driver_version, cuda_version, gpu_count,
|
||||
xid_count, kernel_event_count, rma_count, ecc_info)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
""", (
|
||||
result.filename,
|
||||
result.file_size_mb,
|
||||
result.overall_status,
|
||||
result.total_issues,
|
||||
result.critical_count,
|
||||
result.warning_count,
|
||||
result.info_count,
|
||||
result.driver_version,
|
||||
result.cuda_version,
|
||||
len(result.gpus),
|
||||
len(result.xid_events),
|
||||
len(result.kernel_events),
|
||||
result.rma_count,
|
||||
json.dumps(result.ecc_info, ensure_ascii=False) if result.ecc_info else "{}",
|
||||
))
|
||||
analysis_id = cur.lastrowid
|
||||
|
||||
# GPU 정보 저장
|
||||
for gpu in result.gpus:
|
||||
conn.execute("""
|
||||
INSERT INTO analysis_gpu
|
||||
(analysis_id, gpu_index, name, serial_number, gpu_uuid,
|
||||
pci_address, product_brand, product_architecture,
|
||||
driver_version, cuda_version, vbios_version, inforom_version,
|
||||
temperature, temperature_limit, power_draw, power_limit, power_state,
|
||||
memory_used, memory_total, memory_free,
|
||||
gpu_utilization, memory_utilization,
|
||||
fan_speed, persistence_mode, compute_mode, mig_mode, ecc_mode,
|
||||
pci_link_gen, pci_link_width, clock_graphics, clock_memory,
|
||||
remap_corr_error, remap_uncorr_error, remap_pending, remap_failure,
|
||||
remap_bank_max, remap_bank_high, remap_bank_partial, remap_bank_low, remap_bank_none,
|
||||
sram_corr_volatile, sram_uncorr_parity_volatile, sram_uncorr_secded_volatile,
|
||||
sram_corr_aggregate, sram_uncorr_parity_aggregate, sram_uncorr_secded_aggregate,
|
||||
sram_threshold_exceeded,
|
||||
sram_l2, sram_sm, sram_microcontroller, sram_pcie, sram_other,
|
||||
dram_corr_volatile, dram_uncorr_volatile, dram_corr_aggregate, dram_uncorr_aggregate,
|
||||
hw_slowdown, hw_thermal_slowdown, hw_power_brake_slowdown)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,
|
||||
?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
""", (
|
||||
analysis_id, gpu.index, gpu.name, gpu.serial_number, gpu.gpu_uuid,
|
||||
gpu.pci_address, gpu.product_brand, gpu.product_architecture,
|
||||
gpu.driver_version, gpu.cuda_version, gpu.vbios_version, gpu.inforom_version,
|
||||
gpu.temperature, gpu.temperature_limit, gpu.power_draw, gpu.power_limit, gpu.power_state,
|
||||
gpu.memory_used, gpu.memory_total, gpu.memory_free,
|
||||
gpu.gpu_utilization, gpu.memory_utilization,
|
||||
gpu.fan_speed, gpu.persistence_mode, gpu.compute_mode, gpu.mig_mode, gpu.ecc_mode,
|
||||
gpu.pci_link_gen_current, gpu.pci_link_width_current,
|
||||
gpu.clock_graphics, gpu.clock_memory,
|
||||
gpu.remap_corr_error, gpu.remap_uncorr_error, gpu.remap_pending, gpu.remap_failure,
|
||||
gpu.remap_bank_max, gpu.remap_bank_high, gpu.remap_bank_partial, gpu.remap_bank_low, gpu.remap_bank_none,
|
||||
gpu.sram_corr_volatile, gpu.sram_uncorr_parity_volatile, gpu.sram_uncorr_secded_volatile,
|
||||
gpu.sram_corr_aggregate, gpu.sram_uncorr_parity_aggregate, gpu.sram_uncorr_secded_aggregate,
|
||||
gpu.sram_threshold_exceeded,
|
||||
gpu.sram_l2, gpu.sram_sm, gpu.sram_microcontroller, gpu.sram_pcie, gpu.sram_other,
|
||||
gpu.dram_corr_volatile, gpu.dram_uncorr_volatile, gpu.dram_corr_aggregate, gpu.dram_uncorr_aggregate,
|
||||
gpu.hw_slowdown, gpu.hw_thermal_slowdown, gpu.hw_power_brake_slowdown,
|
||||
))
|
||||
|
||||
# XID 이벤트 저장
|
||||
for xid in result.xid_events:
|
||||
conn.execute("""
|
||||
INSERT INTO analysis_xid
|
||||
(analysis_id, timestamp, xid_code, xid_name, severity,
|
||||
pci_address, gpu_name, gpu_serial, description, action, raw_line)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?)
|
||||
""", (
|
||||
analysis_id, xid.timestamp, xid.xid_code, xid.name, xid.severity,
|
||||
xid.pci_address, xid.gpu_name, xid.gpu_serial, xid.description, xid.action, xid.raw_line,
|
||||
))
|
||||
|
||||
# 커널 이벤트 저장
|
||||
for ke in result.kernel_events:
|
||||
conn.execute("""
|
||||
INSERT INTO analysis_kernel
|
||||
(analysis_id, timestamp, level, message, raw_line)
|
||||
VALUES (?,?,?,?,?)
|
||||
""", (
|
||||
analysis_id, ke.timestamp, ke.level, ke.message, ke.raw_line,
|
||||
))
|
||||
|
||||
# RMA 판정 결과 저장
|
||||
for v in result.rma_verdicts:
|
||||
conn.execute("""
|
||||
INSERT INTO analysis_rma
|
||||
(analysis_id, gpu_index, gpu_name, serial_number,
|
||||
qualifies_for_rma, reasons, warnings,
|
||||
row_remap_details, sram_details)
|
||||
VALUES (?,?,?,?,?,?,?,?,?)
|
||||
""", (
|
||||
analysis_id, v.gpu_index, v.gpu_name, v.serial_number,
|
||||
1 if v.qualifies_for_rma else 0,
|
||||
json.dumps(v.reasons, ensure_ascii=False),
|
||||
json.dumps(v.warnings, ensure_ascii=False),
|
||||
json.dumps(v.row_remap_details, ensure_ascii=False),
|
||||
json.dumps(v.sram_details, ensure_ascii=False),
|
||||
))
|
||||
|
||||
return analysis_id
|
||||
|
||||
|
||||
def update_memo(analysis_id: int, memo: str):
|
||||
"""분석 건에 메모 추가/수정"""
|
||||
with get_conn() as conn:
|
||||
conn.execute("UPDATE analysis SET memo = ? WHERE id = ?", (memo, analysis_id))
|
||||
|
||||
|
||||
def delete_analysis(analysis_id: int):
|
||||
"""분석 건 삭제 (CASCADE로 하위 테이블도 삭제)"""
|
||||
with get_conn() as conn:
|
||||
conn.execute("DELETE FROM analysis WHERE id = ?", (analysis_id,))
|
||||
|
||||
|
||||
def get_analysis_list(limit=100, status_filter=None, search=None):
|
||||
"""분석 이력 목록 조회"""
|
||||
with get_conn() as conn:
|
||||
query = "SELECT * FROM analysis WHERE 1=1"
|
||||
params = []
|
||||
if status_filter and status_filter != "전체":
|
||||
query += " AND overall_status = ?"
|
||||
params.append(status_filter)
|
||||
if search:
|
||||
query += " AND (filename LIKE ? OR memo LIKE ? OR driver_version LIKE ?)"
|
||||
params.extend([f"%{search}%", f"%{search}%", f"%{search}%"])
|
||||
query += " ORDER BY created_at DESC LIMIT ?"
|
||||
params.append(limit)
|
||||
return [dict(row) for row in conn.execute(query, params).fetchall()]
|
||||
|
||||
|
||||
def get_analysis_detail(analysis_id: int):
|
||||
"""분석 건 상세 조회 (GPU, XID, 커널 이벤트 포함)"""
|
||||
with get_conn() as conn:
|
||||
analysis = conn.execute("SELECT * FROM analysis WHERE id = ?", (analysis_id,)).fetchone()
|
||||
if not analysis:
|
||||
return None
|
||||
result = dict(analysis)
|
||||
result["gpus"] = [dict(r) for r in conn.execute(
|
||||
"SELECT * FROM analysis_gpu WHERE analysis_id = ? ORDER BY gpu_index", (analysis_id,)
|
||||
).fetchall()]
|
||||
result["xid_events"] = [dict(r) for r in conn.execute(
|
||||
"SELECT * FROM analysis_xid WHERE analysis_id = ? ORDER BY id", (analysis_id,)
|
||||
).fetchall()]
|
||||
result["kernel_events"] = [dict(r) for r in conn.execute(
|
||||
"SELECT * FROM analysis_kernel WHERE analysis_id = ? ORDER BY id", (analysis_id,)
|
||||
).fetchall()]
|
||||
rma_rows = conn.execute(
|
||||
"SELECT * FROM analysis_rma WHERE analysis_id = ? ORDER BY gpu_index", (analysis_id,)
|
||||
).fetchall()
|
||||
result["rma_verdicts"] = []
|
||||
for r in rma_rows:
|
||||
d = dict(r)
|
||||
d["reasons"] = json.loads(d.get("reasons") or "[]")
|
||||
d["warnings"] = json.loads(d.get("warnings") or "[]")
|
||||
d["row_remap_details"] = json.loads(d.get("row_remap_details") or "{}")
|
||||
d["sram_details"] = json.loads(d.get("sram_details") or "{}")
|
||||
result["rma_verdicts"].append(d)
|
||||
return result
|
||||
|
||||
|
||||
def get_xid_statistics():
|
||||
"""전체 이력에서 XID 코드별 발생 통계"""
|
||||
with get_conn() as conn:
|
||||
return [dict(r) for r in conn.execute("""
|
||||
SELECT xid_code, xid_name, severity,
|
||||
COUNT(*) as total_count,
|
||||
COUNT(DISTINCT analysis_id) as affected_reports,
|
||||
MIN(a.created_at) as first_seen,
|
||||
MAX(a.created_at) as last_seen
|
||||
FROM analysis_xid x
|
||||
JOIN analysis a ON a.id = x.analysis_id
|
||||
GROUP BY xid_code
|
||||
ORDER BY total_count DESC
|
||||
""").fetchall()]
|
||||
|
||||
|
||||
def get_gpu_serial_history(serial_number: str):
|
||||
"""특정 GPU 시리얼 번호의 장애 이력 추적"""
|
||||
with get_conn() as conn:
|
||||
return [dict(r) for r in conn.execute("""
|
||||
SELECT a.id, a.created_at, a.filename, a.overall_status,
|
||||
a.total_issues, a.critical_count, a.memo,
|
||||
g.temperature, g.power_draw, g.memory_used, g.memory_total
|
||||
FROM analysis_gpu g
|
||||
JOIN analysis a ON a.id = g.analysis_id
|
||||
WHERE g.serial_number = ?
|
||||
ORDER BY a.created_at DESC
|
||||
""", (serial_number,)).fetchall()]
|
||||
|
||||
|
||||
def get_all_gpu_serials():
|
||||
"""DB에 기록된 모든 GPU 시리얼 번호 목록"""
|
||||
with get_conn() as conn:
|
||||
return [dict(r) for r in conn.execute("""
|
||||
SELECT DISTINCT serial_number, name, gpu_uuid,
|
||||
COUNT(DISTINCT analysis_id) as report_count
|
||||
FROM analysis_gpu
|
||||
WHERE serial_number != 'N/A'
|
||||
GROUP BY serial_number
|
||||
ORDER BY report_count DESC
|
||||
""").fetchall()]
|
||||
|
||||
|
||||
def get_status_summary():
|
||||
"""전체 분석 이력 상태 요약 (대시보드용)"""
|
||||
with get_conn() as conn:
|
||||
total = conn.execute("SELECT COUNT(*) as cnt FROM analysis").fetchone()["cnt"]
|
||||
by_status = {r["overall_status"]: r["cnt"] for r in conn.execute(
|
||||
"SELECT overall_status, COUNT(*) as cnt FROM analysis GROUP BY overall_status"
|
||||
).fetchall()}
|
||||
recent_critical = [dict(r) for r in conn.execute("""
|
||||
SELECT id, created_at, filename, critical_count, memo
|
||||
FROM analysis WHERE overall_status = 'critical'
|
||||
ORDER BY created_at DESC LIMIT 5
|
||||
""").fetchall()]
|
||||
return {
|
||||
"total_reports": total,
|
||||
"by_status": by_status,
|
||||
"recent_critical": recent_critical,
|
||||
}
|
||||
Reference in New Issue
Block a user