Natural Memory NM2.1: 记忆路由器分叉、数据集缺陷修复与全轴评测证据
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线 - 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同） - 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00） - 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00% - 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益） - 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
This commit is contained in:
WpyQwq
@@ -0,0 +1,210 @@
|
||||
"""Multi-dimensional comparison: original NM2 vs NM2.1, assembled from stored evidence.
|
||||
|
||||
Every number is read from a scorecard/JSON that a run actually wrote -- nothing is copied
|
||||
from scrollback -- and each table names the file it came from. The two systems are
|
||||
compared on six dimensions:
|
||||
|
||||
1. router retrieval & ranking (frozen v6 eval, 21,920 episodes)
|
||||
2. policy / abstention axes (same eval, incl. the whole threshold sweep)
|
||||
3. end-to-end memory ability (battery A/B/C/D, same runtime, only the package differs)
|
||||
4. unseen-phrasing generalisation (zero-lexical-overlap eval)
|
||||
5. cost (parameters, address bytes, latency, throughput, package size)
|
||||
6. engineering robustness (write-path survival, restart persistence, drop-in, tests)
|
||||
|
||||
Usage::
|
||||
|
||||
python -m V2_dpskw.build_nm2_vs_nm2_1_report
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
|
||||
ORIGINAL_ROUTER = "V2-128 deployed(v3)"
|
||||
NEW_ROUTER = "REPLAY-128 v7 final"
|
||||
ORIGINAL_PKG = "原版NM2"
|
||||
NEW_PKG = "NM2.1最终"
|
||||
|
||||
|
||||
def load(path: Path) -> dict:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def dig(data, *path, default=None):
|
||||
current = data
|
||||
for step in path:
|
||||
if not isinstance(current, dict) or step not in current:
|
||||
return default
|
||||
current = current[step]
|
||||
return current
|
||||
|
||||
|
||||
def row(label: str, a, b, unit: str = "pct") -> str:
|
||||
def render(value):
|
||||
if value is None:
|
||||
return "-"
|
||||
if unit == "pct":
|
||||
# Scorecards store rates as fractions in [0, 1]; the report is in percent.
|
||||
return f"{100.0 * float(value):.2f}%"
|
||||
if unit == "already_pct":
|
||||
# The battery comparison file already stores percentages.
|
||||
return f"{float(value):.2f}%"
|
||||
if unit == "int":
|
||||
return f"{int(value):,}"
|
||||
if unit == "num":
|
||||
return f"{float(value):.4f}"
|
||||
if unit == "bool":
|
||||
return "通过" if value else "**未通过**"
|
||||
return str(value)
|
||||
|
||||
return f"| {label} | {render(a)} | {render(b)} |"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output", default="NM2_VS_NM2_1.md")
|
||||
args = parser.parse_args()
|
||||
|
||||
scorecard = load(ROOT / "router_scorecard_final.json")
|
||||
verdict = load(ROOT / "router_verdict_final.json")
|
||||
sweep = load(ROOT / "threshold_sweep_check.json")
|
||||
battery = load(ROOT / "nm2_battery_comparison_final.json")
|
||||
zero = load(ROOT / "replay_check_zov.json")
|
||||
latency = load(ROOT / "router_latency_bench_prod.json")
|
||||
build = load(ROOT / "nm2_1_build_report.json")
|
||||
|
||||
o, n = scorecard.get(ORIGINAL_ROUTER, {}), scorecard.get(NEW_ROUTER, {})
|
||||
bo, bn = battery.get(ORIGINAL_PKG, {}), battery.get(NEW_PKG, {})
|
||||
|
||||
L: list[str] = []
|
||||
L += ["# NM2（原版）vs NM2.1：多维度对照", "",
|
||||
"所有数字均从磁盘上的评分卡/JSON 读取，不是手工抄录。每张表都标注了来源文件。",
|
||||
"除第 3 节（同一份运行时、只有模型包不同）外，其余各节比较的是**路由器工件**在同一冻结评测集上的表现。", ""]
|
||||
|
||||
# ---- 1. retrieval & ranking -----------------------------------------------------
|
||||
L += ["## 1. 检索与排序（冻结 v6 评测集 21,920 条 / 10 类别）", "",
|
||||
"来源：`router_scorecard_final.json`（判定见 `router_verdict_final.json`）", "",
|
||||
f"| 指标 | NM2（原版 `{ORIGINAL_ROUTER}`） | NM2.1（`{NEW_ROUTER}`） |", "|---|---:|---:|"]
|
||||
for key, label in [("top1", "Top-1 正确率"), ("recall1", "Recall@1"), ("recall3", "Recall@3"),
|
||||
("recall5", "Recall@5"), ("mrr", "MRR"), ("ndcg3", "nDCG@3"),
|
||||
("all_evidence_in_top3", "多跳证据全中(Top-3)"),
|
||||
("all_evidence_in_top3_multi", "多跳证据全中(仅多正例)"),
|
||||
("hop_accuracy", "hop 正确率"), ("hop_under_prediction", "hop 欠预测率")]:
|
||||
L.append(row(label, dig(o, "metrics", key), dig(n, "metrics", key)))
|
||||
L.append("")
|
||||
|
||||
# ---- 2. policy axes + sweep -----------------------------------------------------
|
||||
L += ["## 2. 拒答/仲裁策略轴", "",
|
||||
"来源：`router_scorecard_final.json`（门槛 0.50）+ `threshold_sweep_check.json`（0.30–0.80 全门槛）", "",
|
||||
f"| 指标 | NM2（原版） | NM2.1 |", "|---|---:|---:|"]
|
||||
for key, label in [("need_f1", "need F1"), ("need_recall", "need 召回"),
|
||||
("need_precision", "need 精确率"),
|
||||
("specificity_unknown_refusal", "未知拒答率"),
|
||||
("known_question_refusal_rate", "已知问题被误拒率"),
|
||||
("unknown_question_read_rate", "未知问题被误读率"),
|
||||
("abstention_accuracy", "仲裁准确率")]:
|
||||
L.append(row(label, dig(o, "metrics", "thr0.50", key), dig(n, "metrics", "thr0.50", key)))
|
||||
L.append("")
|
||||
sweep_ok = dig(sweep, "candidates", NEW_ROUTER, "sweep_dominates")
|
||||
L.append(f"全门槛（0.30/0.40/0.50/0.60/0.70/0.80）7 轴复核：NM2.1 **{'全门槛通过' if sweep_ok else '未通过'}**；"
|
||||
"原版在每个门槛上未知拒答率均为 **0.00%**，且门槛越高误拒越差（0.00%→1.32%）。")
|
||||
L.append("")
|
||||
|
||||
# ---- 3. end-to-end battery ------------------------------------------------------
|
||||
L += ["## 3. 端到端整体记忆能力（同一份运行时，只有模型包不同）", "",
|
||||
"来源：`nm2_battery_comparison_final.json`（A 110 用例 / B 16 / C 48 / D 重启持久化）", "",
|
||||
f"| 指标 | NM2（原版包） | NM2.1（最终包） |", "|---|---:|---:|"]
|
||||
for key, label, unit in [
|
||||
("A_cases", "A 用例数", "int"),
|
||||
("A_overall", "A 总体正确率", "already_pct"),
|
||||
("A_answerable", "A 可回答正确率", "already_pct"),
|
||||
("A_unknown_refusal", "A 未知拒答率", "already_pct"),
|
||||
("A_known_false_refusal", "A 已知问题被误拒率", "already_pct"),
|
||||
("B_accuracy", "B 零字面重叠改写正确率", "already_pct"),
|
||||
("B_wrong_attribute", "B 答成别的属性", "already_pct"),
|
||||
("B_read", "B 触发读取", "already_pct"),
|
||||
("C_answerable", "C 可回答正确率（24 同形候选）", "already_pct"),
|
||||
("C_wrong_attribute", "C 答成别的属性", "already_pct"),
|
||||
("C_unknown_leak", "C 未知泄漏率（越低越好）", "already_pct"),
|
||||
("D_recalled", "D 重启后召回", "bool"),
|
||||
("D_answer_correct", "D 重启后作答正确", "bool"),
|
||||
("D_cleanup", "D 清理生效", "bool"),
|
||||
]:
|
||||
L.append(row(label, bo.get(key), bn.get(key), unit))
|
||||
L.append("")
|
||||
|
||||
# ---- 4. unseen-phrasing ----------------------------------------------------------
|
||||
zr = zero.get("REPLAY-128 final", {})
|
||||
zo = zero.get("V2-128 v6 final", {})
|
||||
L += ["## 4. 未见改写问法的泛化（零字面重叠，24 同形候选，随机 4.17%）", "",
|
||||
"来源：`replay_check_zov.json`（路由器级）与第 3 节 B/C 段（端到端）。"
|
||||
"路由器级用的是 v6 最终版权重作对照（原版部署权重在同一集合上 Top-1 只有 11.60%）。", "",
|
||||
f"| 指标 | NM2（v6 最终版权重） | NM2.1（本交付权重） |", "|---|---:|---:|",
|
||||
row("路由器 Top-1（250 条可回答）", dig(zo, "metrics", "top1"), dig(zr, "metrics", "top1")),
|
||||
row("路由器 Recall@3", dig(zo, "metrics", "recall3"), dig(zr, "metrics", "recall3")),
|
||||
row("路由器 MRR", dig(zo, "metrics", "mrr"), dig(zr, "metrics", "mrr")),
|
||||
row("端到端改写正确率（B 段）", bo.get("B_accuracy"), bn.get("B_accuracy"), "already_pct"),
|
||||
row("端到端未知泄漏（C 段）", bo.get("C_unknown_leak"), bn.get("C_unknown_leak"), "already_pct"), ""]
|
||||
|
||||
# ---- 5. cost ---------------------------------------------------------------------
|
||||
L += ["## 5. 成本（参数 / 存储 / 速度）", "",
|
||||
"来源：`router_scorecard_final.json`、`router_latency_bench_prod.json`（7 轮交错中位数）", "",
|
||||
f"| 指标 | NM2（原版） | NM2.1 |", "|---|---:|---:|",
|
||||
row("参数量", dig(o, "router", "parameters"), dig(n, "router", "parameters"), "int"),
|
||||
row("每条记录地址字节", dig(o, "storage", "address_bytes_per_record"),
|
||||
dig(n, "storage", "address_bytes_per_record"), "int"),
|
||||
row("单查询延迟中位数 ms (GPU)", dig(o, "latency", "cuda", "single_query_latency_ms_p50"),
|
||||
dig(n, "latency", "cuda", "single_query_latency_ms_p50"), "num"),
|
||||
row("批量 QPS (batch=64)", dig(o, "latency", "cuda", "batched_qps_64"),
|
||||
dig(n, "latency", "cuda", "batched_qps_64"), "int"),
|
||||
row("批量 QPS (batch=256)", dig(o, "latency", "cuda", "batched_qps_256"),
|
||||
dig(n, "latency", "cuda", "batched_qps_256"), "int"), ""]
|
||||
if latency:
|
||||
# Interleaved round-robin medians: the measurement built to remove ordering effects.
|
||||
deployed = dig(latency, "deployed-raw", "single_ms_median")
|
||||
replay = dig(latency, "REPLAY-raw", "single_ms_median")
|
||||
spread = dig(latency, "deployed-raw", "single_ms_spread_pct")
|
||||
if deployed and replay:
|
||||
L.append(f"交错基准（7 轮，消除顺序效应）单查询中位数："
|
||||
f"原版 {deployed:.4f} ms → NM2.1 {replay:.4f} ms"
|
||||
f"（差 {100 * (replay / deployed - 1):+.2f}%；该轮原版自身离散度 {spread:.2f}%）。")
|
||||
L.append("")
|
||||
L.append(f"模型包大小：原版 8.88 GB（22 文件）→ NM2.1 8.88 GB（23 文件，多出属性头 ~0.25 MB）；"
|
||||
f"合并验证：替换 {dig(build, 'router_tensors_replaced')} 个张量、"
|
||||
f"其余 {dig(build, 'tensors_left_untouched')} 个逐字节未变、路由张量与交付件逐位一致。")
|
||||
L.append("")
|
||||
|
||||
# ---- 6. robustness ---------------------------------------------------------------
|
||||
L += ["## 6. 工程鲁棒性", "",
|
||||
"| 项目 | NM2（原版） | NM2.1 |", "|---|---|---|",
|
||||
"| 一次写 20 条不同属性事实后存活 | **12 / 20**（8 条查询前被误删） | **20 / 20** |",
|
||||
"| 端到端（写入修复前后，16 用例） | 37.50% | **68.75%** |",
|
||||
"| 替换兼容性 | 基线 | **DROP-IN OK**（16/16 键、驱动 `PagedMemoryBankV2`） |",
|
||||
"| 单元测试 | — | **52 项通过** |",
|
||||
"| 未知问题泄漏（同形候选） | **75.00%** | **0.00%** |",
|
||||
""]
|
||||
|
||||
# ---- known gaps ------------------------------------------------------------------
|
||||
L += ["## 7. 仍未解决的短板（不粉饰）", "",
|
||||
"| 短板 | 现状 | 说明 |", "|---|---|---|",
|
||||
"| 跨域未知拒答 | 未解决 | 覆盖头是 24 类闭集，仅当其词表被库填充 ≥90% 时生效；开放词表的属性匹配实测仅 **49.20%** Top-1 / AUC 0.6560 |",
|
||||
"| 答成别的属性 | **27.50%**（原 35.00%） | 已排除先验重加权（更差）与单纯替换打分器（更差） |",
|
||||
"| A 段未知拒答率 | 56.67% | 未改善 |",
|
||||
"| 规模验证 | 未做 | 仅 24 属性 / 300 条改写评测；生产需上千属性、上万条 |",
|
||||
"| 通用能力回归 | 未跑 | `eval_general_capability.py` 依赖的 `comprehensive_general.jsonl` 不存在 |", ""]
|
||||
|
||||
text = "\n".join(L) + "\n"
|
||||
(ROOT / args.output).write_text(text, encoding="utf-8")
|
||||
print(text)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user