Natural Memory NM2.1: 记忆路由器分叉、数据集缺陷修复与全轴评测证据
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线 - 修复 prepare_memory_router_dataset 候选池重建缺陷(mega 家族 3568x 加速,输出逐字节相同) - 修复 v5 被破坏的拒答与多跳标签(train 未知样本 319 -> 16319,multi_hop 平均正例 1.00 -> 2.00) - 同存储预算下 V2-128 v6 逐轴 22/22 通过:Top-1 41.12% -> 94.62%,未知拒答 0.00% -> 100.00% - 记录三条被实测推翻的显然优化(logits_to_keep=1 反而慢 55%、XL 容量未带来收益) - 记忆手术跨架构可移植性 14/14,读写关闭时与原生模型逐位相同
This commit is contained in:
@@ -0,0 +1,58 @@
|
||||
"""Reply-level diff between two realistic_v2 eval runs, per category."""
|
||||
import json
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load(path):
|
||||
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
key = next(iter(data))
|
||||
body = data[key]
|
||||
rows = body if isinstance(body, list) else body.get("rows", [])
|
||||
return rows
|
||||
|
||||
|
||||
def idx_keys(rows):
|
||||
idx = defaultdict(int)
|
||||
out = []
|
||||
for r in rows:
|
||||
c = r["category"]
|
||||
out.append((c, r["query"], idx[c]))
|
||||
idx[c] += 1
|
||||
return out
|
||||
|
||||
|
||||
def main(a, b):
|
||||
ra, rb = load(a), load(b)
|
||||
ka, kb = idx_keys(ra), idx_keys(rb)
|
||||
ma = {k: r for k, r in zip(ka, ra)}
|
||||
mb = {k: r for k, r in zip(kb, rb)}
|
||||
same_reply = defaultdict(int)
|
||||
diff_reply = defaultdict(int)
|
||||
diff_examples = defaultdict(list)
|
||||
for k in ma:
|
||||
if k not in mb:
|
||||
continue
|
||||
if ma[k].get("reply") == mb[k].get("reply"):
|
||||
same_reply[k[0]] += 1
|
||||
else:
|
||||
diff_reply[k[0]] += 1
|
||||
if len(diff_examples[k[0]]) < 3:
|
||||
diff_examples[k[0]].append((k[1], ma[k].get("reply"), mb[k].get("reply")))
|
||||
cats = sorted(set(same_reply) | set(diff_reply))
|
||||
print(f"{'category':<20}{'same':>6}{'changed':>9}{'total':>7}")
|
||||
for c in cats:
|
||||
print(f"{c:<20}{same_reply[c]:>6}{diff_reply[c]:>9}{same_reply[c]+diff_reply[c]:>7}")
|
||||
print()
|
||||
for c in cats:
|
||||
if diff_reply[c]:
|
||||
print(f"=== {c}: {diff_reply[c]} replies changed ===")
|
||||
for q, x, y in diff_examples[c]:
|
||||
print(f" Q: {q}")
|
||||
print(f" old: {x!r}")
|
||||
print(f" new: {y!r}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1], sys.argv[2])
|
||||
Reference in New Issue
Block a user