"""Multi-dimensional comparison: original NM2 vs NM2.1, assembled from stored evidence. Every number is read from a scorecard/JSON that a run actually wrote -- nothing is copied from scrollback -- and each table names the file it came from. The two systems are compared on six dimensions: 1. router retrieval & ranking (frozen v6 eval, 21,920 episodes) 2. policy / abstention axes (same eval, incl. the whole threshold sweep) 3. end-to-end memory ability (battery A/B/C/D, same runtime, only the package differs) 4. unseen-phrasing generalisation (zero-lexical-overlap eval) 5. cost (parameters, address bytes, latency, throughput, package size) 6. engineering robustness (write-path survival, restart persistence, drop-in, tests) Usage:: python -m V2_dpskw.build_nm2_vs_nm2_1_report """ from __future__ import annotations import argparse import json from pathlib import Path ROOT = Path(__file__).resolve().parent ORIGINAL_ROUTER = "V2-128 deployed(v3)" NEW_ROUTER = "REPLAY-128 v7 final" ORIGINAL_PKG = "原版NM2" NEW_PKG = "NM2.1最终" def load(path: Path) -> dict: try: return json.loads(path.read_text(encoding="utf-8")) except Exception: return {} def dig(data, *path, default=None): current = data for step in path: if not isinstance(current, dict) or step not in current: return default current = current[step] return current def row(label: str, a, b, unit: str = "pct") -> str: def render(value): if value is None: return "-" if unit == "pct": # Scorecards store rates as fractions in [0, 1]; the report is in percent. return f"{100.0 * float(value):.2f}%" if unit == "already_pct": # The battery comparison file already stores percentages. return f"{float(value):.2f}%" if unit == "int": return f"{int(value):,}" if unit == "num": return f"{float(value):.4f}" if unit == "bool": return "通过" if value else "**未通过**" return str(value) return f"| {label} | {render(a)} | {render(b)} |" def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--output", default="NM2_VS_NM2_1.md") args = parser.parse_args() scorecard = load(ROOT / "router_scorecard_final.json") verdict = load(ROOT / "router_verdict_final.json") sweep = load(ROOT / "threshold_sweep_check.json") battery = load(ROOT / "nm2_battery_comparison_final.json") zero = load(ROOT / "replay_check_zov.json") latency = load(ROOT / "router_latency_bench_prod.json") build = load(ROOT / "nm2_1_build_report.json") o, n = scorecard.get(ORIGINAL_ROUTER, {}), scorecard.get(NEW_ROUTER, {}) bo, bn = battery.get(ORIGINAL_PKG, {}), battery.get(NEW_PKG, {}) L: list[str] = [] L += ["# NM2（原版）vs NM2.1：多维度对照", "", "所有数字均从磁盘上的评分卡/JSON 读取，不是手工抄录。每张表都标注了来源文件。", "除第 3 节（同一份运行时、只有模型包不同）外，其余各节比较的是**路由器工件**在同一冻结评测集上的表现。", ""] # ---- 1. retrieval & ranking ----------------------------------------------------- L += ["## 1. 检索与排序（冻结 v6 评测集 21,920 条 / 10 类别）", "", "来源：`router_scorecard_final.json`（判定见 `router_verdict_final.json`）", "", f"| 指标 | NM2（原版 `{ORIGINAL_ROUTER}`） | NM2.1（`{NEW_ROUTER}`） |", "|---|---:|---:|"] for key, label in [("top1", "Top-1 正确率"), ("recall1", "Recall@1"), ("recall3", "Recall@3"), ("recall5", "Recall@5"), ("mrr", "MRR"), ("ndcg3", "nDCG@3"), ("all_evidence_in_top3", "多跳证据全中(Top-3)"), ("all_evidence_in_top3_multi", "多跳证据全中(仅多正例)"), ("hop_accuracy", "hop 正确率"), ("hop_under_prediction", "hop 欠预测率")]: L.append(row(label, dig(o, "metrics", key), dig(n, "metrics", key))) L.append("") # ---- 2. policy axes + sweep ----------------------------------------------------- L += ["## 2. 拒答/仲裁策略轴", "", "来源：`router_scorecard_final.json`（门槛 0.50）+ `threshold_sweep_check.json`（0.30–0.80 全门槛）", "", f"| 指标 | NM2（原版） | NM2.1 |", "|---|---:|---:|"] for key, label in [("need_f1", "need F1"), ("need_recall", "need 召回"), ("need_precision", "need 精确率"), ("specificity_unknown_refusal", "未知拒答率"), ("known_question_refusal_rate", "已知问题被误拒率"), ("unknown_question_read_rate", "未知问题被误读率"), ("abstention_accuracy", "仲裁准确率")]: L.append(row(label, dig(o, "metrics", "thr0.50", key), dig(n, "metrics", "thr0.50", key))) L.append("") sweep_ok = dig(sweep, "candidates", NEW_ROUTER, "sweep_dominates") L.append(f"全门槛（0.30/0.40/0.50/0.60/0.70/0.80）7 轴复核：NM2.1 **{'全门槛通过' if sweep_ok else '未通过'}**；" "原版在每个门槛上未知拒答率均为 **0.00%**，且门槛越高误拒越差（0.00%→1.32%）。") L.append("") # ---- 3. end-to-end battery ------------------------------------------------------ L += ["## 3. 端到端整体记忆能力（同一份运行时，只有模型包不同）", "", "来源：`nm2_battery_comparison_final.json`（A 110 用例 / B 16 / C 48 / D 重启持久化）", "", f"| 指标 | NM2（原版包） | NM2.1（最终包） |", "|---|---:|---:|"] for key, label, unit in [ ("A_cases", "A 用例数", "int"), ("A_overall", "A 总体正确率", "already_pct"), ("A_answerable", "A 可回答正确率", "already_pct"), ("A_unknown_refusal", "A 未知拒答率", "already_pct"), ("A_known_false_refusal", "A 已知问题被误拒率", "already_pct"), ("B_accuracy", "B 零字面重叠改写正确率", "already_pct"), ("B_wrong_attribute", "B 答成别的属性", "already_pct"), ("B_read", "B 触发读取", "already_pct"), ("C_answerable", "C 可回答正确率（24 同形候选）", "already_pct"), ("C_wrong_attribute", "C 答成别的属性", "already_pct"), ("C_unknown_leak", "C 未知泄漏率（越低越好）", "already_pct"), ("D_recalled", "D 重启后召回", "bool"), ("D_answer_correct", "D 重启后作答正确", "bool"), ("D_cleanup", "D 清理生效", "bool"), ]: L.append(row(label, bo.get(key), bn.get(key), unit)) L.append("") # ---- 4. unseen-phrasing ---------------------------------------------------------- zr = zero.get("REPLAY-128 final", {}) zo = zero.get("V2-128 v6 final", {}) L += ["## 4. 未见改写问法的泛化（零字面重叠，24 同形候选，随机 4.17%）", "", "来源：`replay_check_zov.json`（路由器级）与第 3 节 B/C 段（端到端）。" "路由器级用的是 v6 最终版权重作对照（原版部署权重在同一集合上 Top-1 只有 11.60%）。", "", f"| 指标 | NM2（v6 最终版权重） | NM2.1（本交付权重） |", "|---|---:|---:|", row("路由器 Top-1（250 条可回答）", dig(zo, "metrics", "top1"), dig(zr, "metrics", "top1")), row("路由器 Recall@3", dig(zo, "metrics", "recall3"), dig(zr, "metrics", "recall3")), row("路由器 MRR", dig(zo, "metrics", "mrr"), dig(zr, "metrics", "mrr")), row("端到端改写正确率（B 段）", bo.get("B_accuracy"), bn.get("B_accuracy"), "already_pct"), row("端到端未知泄漏（C 段）", bo.get("C_unknown_leak"), bn.get("C_unknown_leak"), "already_pct"), ""] # ---- 5. cost --------------------------------------------------------------------- L += ["## 5. 成本（参数 / 存储 / 速度）", "", "来源：`router_scorecard_final.json`、`router_latency_bench_prod.json`（7 轮交错中位数）", "", f"| 指标 | NM2（原版） | NM2.1 |", "|---|---:|---:|", row("参数量", dig(o, "router", "parameters"), dig(n, "router", "parameters"), "int"), row("每条记录地址字节", dig(o, "storage", "address_bytes_per_record"), dig(n, "storage", "address_bytes_per_record"), "int"), row("单查询延迟中位数 ms (GPU)", dig(o, "latency", "cuda", "single_query_latency_ms_p50"), dig(n, "latency", "cuda", "single_query_latency_ms_p50"), "num"), row("批量 QPS (batch=64)", dig(o, "latency", "cuda", "batched_qps_64"), dig(n, "latency", "cuda", "batched_qps_64"), "int"), row("批量 QPS (batch=256)", dig(o, "latency", "cuda", "batched_qps_256"), dig(n, "latency", "cuda", "batched_qps_256"), "int"), ""] if latency: # Interleaved round-robin medians: the measurement built to remove ordering effects. deployed = dig(latency, "deployed-raw", "single_ms_median") replay = dig(latency, "REPLAY-raw", "single_ms_median") spread = dig(latency, "deployed-raw", "single_ms_spread_pct") if deployed and replay: L.append(f"交错基准（7 轮，消除顺序效应）单查询中位数：" f"原版 {deployed:.4f} ms → NM2.1 {replay:.4f} ms" f"（差 {100 * (replay / deployed - 1):+.2f}%；该轮原版自身离散度 {spread:.2f}%）。") L.append("") L.append(f"模型包大小：原版 8.88 GB（22 文件）→ NM2.1 8.88 GB（23 文件，多出属性头 ~0.25 MB）；" f"合并验证：替换 {dig(build, 'router_tensors_replaced')} 个张量、" f"其余 {dig(build, 'tensors_left_untouched')} 个逐字节未变、路由张量与交付件逐位一致。") L.append("") # ---- 6. robustness --------------------------------------------------------------- L += ["## 6. 工程鲁棒性", "", "| 项目 | NM2（原版） | NM2.1 |", "|---|---|---|", "| 一次写 20 条不同属性事实后存活 | **12 / 20**（8 条查询前被误删） | **20 / 20** |", "| 端到端（写入修复前后，16 用例） | 37.50% | **68.75%** |", "| 替换兼容性 | 基线 | **DROP-IN OK**（16/16 键、驱动 `PagedMemoryBankV2`） |", "| 单元测试 | — | **52 项通过** |", "| 未知问题泄漏（同形候选） | **75.00%** | **0.00%** |", ""] # ---- known gaps ------------------------------------------------------------------ L += ["## 7. 仍未解决的短板（不粉饰）", "", "| 短板 | 现状 | 说明 |", "|---|---|---|", "| 跨域未知拒答 | 未解决 | 覆盖头是 24 类闭集，仅当其词表被库填充 ≥90% 时生效；开放词表的属性匹配实测仅 **49.20%** Top-1 / AUC 0.6560 |", "| 答成别的属性 | **27.50%**（原 35.00%） | 已排除先验重加权（更差）与单纯替换打分器（更差） |", "| A 段未知拒答率 | 56.67% | 未改善 |", "| 规模验证 | 未做 | 仅 24 属性 / 300 条改写评测；生产需上千属性、上万条 |", "| 通用能力回归 | 未跑 | `eval_general_capability.py` 依赖的 `comprehensive_general.jsonl` 不存在 |", ""] text = "\n".join(L) + "\n" (ROOT / args.output).write_text(text, encoding="utf-8") print(text) return 0 if __name__ == "__main__": raise SystemExit(main())