WpyQwq
643e22ecb9
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线 - 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同） - 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00） - 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00% - 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益） - 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
63 lines
2.3 KiB
Python
63 lines
2.3 KiB
Python
"""Prove that stored run rows pair correctly with corpus cases.
|
|
|
|
Pairing is by position (``build_cases`` emits categories in sorted order, file
|
|
order inside a category), so any drift between the corpus as it is *now* and the
|
|
corpus as it was *when the run happened* would silently pair the wrong anchor
|
|
with the wrong reply -- and repeated queries inside a category would hide it.
|
|
|
|
The stored rows carry ``matched``, the list the legacy scorer produced at run
|
|
time. Recomputing it from the paired case must reproduce that list exactly;
|
|
that is an independent witness of correct pairing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
from V2_dpskw.rescore_e2e import load_corpus, load_run
|
|
|
|
|
|
def legacy_accept(reply: str, case: dict) -> list[str]:
|
|
lowered = reply.lower()
|
|
return [v for v in case["acceptable"] if v and v.lower() in lowered][:3]
|
|
|
|
|
|
def check(corpus_path: Path, run_path: Path, per_category: int = 25) -> bool:
|
|
cases = load_corpus(corpus_path, per_category)
|
|
rows = load_run(run_path)
|
|
print(f"{run_path.name}: {len(rows)} rows / {len(cases)} cases")
|
|
if len(rows) != len(cases):
|
|
print(" !! length mismatch")
|
|
return False
|
|
bad = []
|
|
for i, (row, case) in enumerate(zip(rows, cases)):
|
|
expect = legacy_accept(row.get("reply", ""), case)
|
|
got = row.get("matched") or []
|
|
if expect != got:
|
|
bad.append((i, case, row, expect, got))
|
|
if not bad:
|
|
print(" pairing VERIFIED: every stored 'matched' field reproduces exactly")
|
|
return True
|
|
print(f" !! {len(bad)} rows disagree with the corpus")
|
|
per_cat = defaultdict(int)
|
|
for _, case, _, _, _ in bad:
|
|
per_cat[case["category"]] += 1
|
|
print(f" disagreement by category: {dict(per_cat)}")
|
|
for i, case, row, expect, got in bad[:6]:
|
|
print(f" row#{i} cat={case['category']}")
|
|
print(f" corpus query={case['query']!r} acceptable={case['acceptable']!r}")
|
|
print(f" row query={row.get('query')!r}")
|
|
print(f" expect matched={expect!r} stored={got!r}")
|
|
return False
|
|
|
|
|
|
if __name__ == "__main__":
|
|
ok = True
|
|
for run in sys.argv[2:]:
|
|
ok &= check(Path(sys.argv[1]), Path(run))
|
|
print("\nALL PAIRINGS OK" if ok else "\nPAIRING PROBLEM")
|