wpy/natural-memory-nm21
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Natural Memory NM2.1: 记忆路由器分叉、数据集缺陷修复与全轴评测证据

Browse Source 
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线
- 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同）
- 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00）
- 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00%
- 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益）
- 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
This commit is contained in:
WpyQwq
2026-09-19 11:11:31 +08:00
commit 643e22ecb9
484 changed files with 306821 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
make_realistic_memory_eval.py
+299
View File
@@ -0,0 +1,299 @@
"""Build a *realistic-shaped* memory evaluation corpus.
Why: every number this project has produced so far rests on a template corpus -- one
sentence shape ("我的X是 VAL-…。"), a single entity (user), 24 attributes and no updates,
conflicts, multi-hop chains or noise. Those numbers are internally consistent and
reproducible, but none of them predict behaviour on text that looks like a real user's.
This generator produces the missing population.
Eight categories, each generating the same episode schema the existing harnesses consume
(``query`` / ``candidates[].text`` / ``positive_indices`` / ``metadata.acceptable``):
multi_entity 同一属性在多个实体上出现，问题只问其中一个
alias_paraphrase 提问与事实无字面重叠（换一种说法）
update_conflict 同一实体+属性写过两次，正确答案是较新那个，旧值是强干扰
multi_hop 答案需要串联两条事实（两条都是正例）
near_miss 候选属性名高度相近（手机尾号 / 座机尾号）
noise_context 库里混入大量无关闲聊，问题仍然可回答
long_fact 事实是长句，值嵌在句子中间
unknown_attribute 问一个从未写过的属性（不可回答，必须拒答）
Usage::
python -m V2_dpskw.make_realistic_memory_eval --output data/realistic_eval.jsonl
"""
from __future__ import annotations
import argparse
import hashlib
import json
import random
from pathlib import Path
#: entity -> (subject phrase used inside facts, alias list used by questions)
ENTITIES = [
("user", ["我", "我本人"], ["我", "我自己"]),
("zhangsan", ["张三", "我们组的张三"], ["张三", "我们组那位张三"]),
("liworks", ["李工", "运维的李工"], ["李工", "运维那位李工"]),
("proj_alpha", ["项目 Alpha", "Alpha 项目"], ["Alpha", "Alpha 项目"]),
("client_beta", ["客户 Beta", "Beta 客户"], ["Beta", "Beta 客户"]),
]
#: attribute -> (fact frames, question frames without lexical overlap)
ATTRIBUTES = [
("值班电话", ["{s}的值班电话是 {v}。", "{s}留的值班号码为 {v}。"],
["{q}有急事按哪个号找人？", "{q}轮班时该拨什么号码？"]),
("常驻机房", ["{s}常驻机房在 {v}。", "{s}平时待的机房是 {v}。"],
["{q}平时在哪间屋子干活？", "{q}固定在哪个位置办公？"]),
("发布窗口", ["{s}的发布窗口是 {v}。", "{s}固定 {v} 做上线。"],
["{q}一般什么时候做上线？", "{q}挑哪个时段变更？"]),
("接口版本号", ["{s}对接的接口版本号是 {v}。", "{s}目前跑的是 {v} 版本接口。"],
["{q}现在对接的是哪一版？", "{q}联调用的那个版本是什么？"]),
("告警阈值", ["{s}的告警阈值是 {v}。", "{s}超过 {v} 就报警。"],
["{q}到多少会触发报警？", "{q}哪条线一破就叫？"]),
("应急联系人", ["{s}的应急联系人是 {v}。", "{s}出事找 {v}。"],
["{q}出问题该喊谁？", "{q}紧急情况下找谁接手？"]),
("主库地址", ["{s}的主库地址是 {v}。", "{s}主库落在 {v}。"],
["{q}数据落在哪一个地址？", "{q}写库连的是哪里？"]),
("缓存容量", ["{s}的缓存容量是 {v}。", "{s}缓存开到 {v}。"],
["{q}内存开到多大？", "{q}缓冲配了多少？"]),
]
#: attribute pairs that are deliberately easy to confuse
NEAR_MISS = [
("手机尾号", "座机尾号"),
("办公城市", "常住城市"),
("工位楼层", "宿舍楼层"),
("项目代号", "客户代号"),
]
_VALUES = ["分机 8821", "B3-204", "每周三 22:00", "v3.14.2", "91.5%", "王工 138****6621",
"10.20.3.7:5432", "64GB", "A-7719", "夜间 02:00", "v2.8.0", "78%",
"李工 139****3344", "10.20.9.1:6379", "128GB", "B-3310"]
_ALIAS_FACTS = [
("我平时待得最久的那座城是 {v}。", "我常年落脚在哪儿？"),
("我每天睡醒最早看的那个数字是 {v}。", "我那个数字是多少？"),
("我手上那台机器是 {v}。", "我用的那台是什么款？"),
("我每天早上灌下去的那杯偏爱 {v}。", "我早上那杯是什么口味？"),
]
_QUESTION_POOL = ["{q}这个是什么？", "{q}对应的值是多少？"]
def _value(rng: random.Random) -> str:
return rng.choice(_VALUES)
def _episode(index: int, category: str, query: str, candidates: list[str],
positive_texts: list[str], acceptable: list[str], *, hop: int = 1,
extra: dict | None = None) -> dict:
positives = [position for position, text in enumerate(candidates) if text in positive_texts]
metadata = {
"category": category,
"answerable": bool(positives),
"acceptable": acceptable if positives else [],
"hop_count": hop if positives else 0,
"answer": acceptable[0] if acceptable and positives else "",
}
if extra:
metadata.update(extra)
return {
"id": "realistic-%s-%05d" % (category, index),
"group_id": "realistic-%s-%05d" % (category, index),
"source": "realistic_memory_eval",
"family": category,
"query": query,
"candidates": [{"text": text, "kind": "fact", "entity": "user", "attribute": ""}
for text in candidates],
"positive_indices": positives,
"positive_index": positives[0] if positives else -1,
"need_memory": 1.0 if positives else 0.0,
"hop": hop if positives else 0,
"metadata": metadata,
}
def build(count_per_category: int, seed: int) -> list[dict]:
rng = random.Random(seed)
episodes: list[dict] = []
counter = 0
for _ in range(count_per_category):
# --- multi_entity: same attribute on several entities, one is asked about -----
attribute, fact_frames, question_frames = rng.choice(ATTRIBUTES)
speakers = rng.sample(ENTITIES, 3)
target = rng.choice(speakers)
candidates, values = [], {}
for key, subjects, aliases in speakers:
value = _value(rng)
values[key] = value
candidates.append(rng.choice(fact_frames).format(s=subjects[0], v=value))
rng.shuffle(candidates)
fact = rng.choice(fact_frames).format(s=target[1][0], v=values[target[0]])
# Rebuild so the target's own phrasing is present verbatim among candidates.
candidates = []
for key, subjects, aliases in speakers:
frame = fact_frames[0] if key == target[0] else fact_frames[1 % len(fact_frames)]
candidates.append(frame.format(s=subjects[0], v=values[key]))
rng.shuffle(candidates)
fact = [c for c in candidates if values[target[0]] in c][0]
query = rng.choice(question_frames).format(q=rng.choice(target[2]))
episodes.append(_episode(counter, "multi_entity", query, candidates, [fact],
[values[target[0]]], extra={"attribute": attribute}))
counter += 1
# --- alias_paraphrase: fact and question share no distinctive characters -------
# The target fact is the ONLY fact written in its own frame; every distractor uses
# a different attribute's frame, so the question identifies exactly one value.
# (An earlier version wrote the same frame with five different values, which is not
# a paraphrase test at all but a self-contradictory bank.)
target_index = rng.randrange(len(ATTRIBUTES))
template, question = rng.choice(_ALIAS_FACTS)
value = _value(rng)
fact = template.format(v=value)
others = [ATTRIBUTES[i][1][0].format(s="我", v=_value(rng))
for i in range(len(ATTRIBUTES)) if i != target_index][:4]
candidates = [fact] + others
rng.shuffle(candidates)
episodes.append(_episode(counter, "alias_paraphrase", question, candidates, [fact], [value]))
counter += 1
# --- update_conflict: older value is the strong distractor ---------------------
attribute, fact_frames, question_frames = rng.choice(ATTRIBUTES)
old_value, new_value = _value(rng), _value(rng)
while new_value == old_value:
new_value = _value(rng)
old_fact = fact_frames[0].format(s="我", v=old_value)
new_fact = fact_frames[1 % len(fact_frames)].format(s="我", v=new_value)
candidates = [old_fact, new_fact]
candidates += [fact_frames[0].format(s="我", v=_value(rng)) for _ in range(3)]
rng.shuffle(candidates)
query = rng.choice(question_frames).format(q="我")
episodes.append(_episode(counter, "update_conflict", query, candidates, [new_fact],
[new_value], extra={"attribute": attribute,
"superseded_value": old_value}))
counter += 1
# --- multi_hop: the answer needs two chained facts, and is a *unique* token ------
# The answer must not be recoverable by blending: every chain points at a distinct
# code, so only following the right chain yields the expected token.
code = "REF-%s" % "".join(rng.choice("ABCDEFGHJKLMNPQRSTUVWXYZ23456789") for _ in range(6))
answer = "值班人-7741"
others = ["值班人-%04d" % rng.randrange(1000, 9999) for _ in range(3)]
first = "张三负责的那个项目的内部编号是 %s。" % code
second = "编号 %s 对应的值班人是 %s。" % (code, answer)
decoy_codes = ["REF-" + "".join(rng.choice("ABCDEFGHJKLMNPQRSTUVWXYZ23456789")
for _ in range(6)) for _ in others]
decoys = ["编号 %s 对应的值班人是 %s。" % (dc, name)
for dc, name in zip(decoy_codes, others)]
candidates = [first, second] + decoys
rng.shuffle(candidates)
query = "负责那个项目的同事对应的值班人是谁？"
episodes.append(_episode(counter, "multi_hop", query, candidates, [first, second],
[answer], hop=2))
counter += 1
# --- near_miss: the question refers to the attribute without naming it ----------
# Naming the attribute outright ("我的手机尾号是什么") is not a hard test; the
# question here paraphrases it so the near-miss decoy is genuinely in play.
left, right = rng.choice(NEAR_MISS)
value, decoy = _value(rng), _value(rng)
candidates = ["我的%s是 %s。" % (left, value), "我的%s是 %s。" % (right, decoy)]
candidates += ["我的%s是 %s。" % (name, _value(rng))
for name, _, _ in rng.sample(ATTRIBUTES, 2)]
rng.shuffle(candidates)
ask_left = rng.random() < 0.5
target_value = value if ask_left else decoy
target_fact = [c for c in candidates if target_value in c][0]
query = ("我随身那台设备的末几位数字是啥？" if ask_left
else "我家座机最后几位数字是啥？")
episodes.append(_episode(counter, "near_miss", query, candidates, [target_fact],
[target_value], extra={"attribute": left if ask_left else right}))
counter += 1
# --- noise_context: lots of unrelated chatter, question still answerable -------
attribute, fact_frames, question_frames = rng.choice(ATTRIBUTES)
value = _value(rng)
fact = fact_frames[0].format(s="我", v=value)
noise = ["这是普通对话噪声：验证用户-%06d 暂时提到一个无关编号 N-%s，不需要长期保存。"
% (rng.randrange(10 ** 6), "".join(rng.choice("ABCDEFGHJKLMNPQRSTUVWXYZ23456789")
for _ in range(8)))
for _ in range(8)]
candidates = [fact] + noise
rng.shuffle(candidates)
query = rng.choice(question_frames).format(q="我")
episodes.append(_episode(counter, "noise_context", query, candidates, [fact], [value]))
counter += 1
# --- long_fact: the value sits mid-sentence in a long fact ---------------------
attribute, fact_frames, question_frames = rng.choice(ATTRIBUTES)
value = _value(rng)
long_fact = ("关于%s这件事，之前散会时我们临时定了下来，当时讨论得比较久，最后确认%s，"
"后续如果还有调整会在群里同步，暂时先按这个执行。" % (attribute, value))
others = [("关于%s这件事，会上也提过一嘴，但那次的说法是%s，后来没有正式确认过。"
% (attribute, _value(rng))) for _ in range(3)]
candidates = [long_fact] + others
rng.shuffle(candidates)
query = rng.choice(question_frames).format(q="我")
episodes.append(_episode(counter, "long_fact", query, candidates, [long_fact], [value]))
counter += 1
# --- unknown_attribute: the asked attribute was never written ------------------
# Candidates must be written in *their own* frames (an earlier version reused the
# asked attribute's frame, so the "unknown" attribute was in fact present and the
# question was answerable -- the 0.00% it produced measured nothing).
attribute, fact_frames, question_frames = rng.choice(ATTRIBUTES)
written = [(name, frames) for name, frames, _ in ATTRIBUTES if name != attribute]
picked = rng.sample(written, 5)
candidates = [frames[0].format(s="我", v=_value(rng)) for _, frames in picked]
query = rng.choice(question_frames).format(q="我")
episodes.append(_episode(counter, "unknown_attribute", query, candidates, [], []))
counter += 1
return episodes
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", default="data/realistic_eval.jsonl")
parser.add_argument("--per-category", type=int, default=25)
parser.add_argument("--seed", type=int, default=20260913)
args = parser.parse_args()
episodes = build(args.per_category, args.seed)
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
with output.open("w", encoding="utf-8") as handle:
for row in episodes:
handle.write(json.dumps(row, ensure_ascii=False, separators=(",", ":")) + "\n")
counts: dict[str, dict] = {}
for row in episodes:
category = row["metadata"]["category"]
bucket = counts.setdefault(category, {"episodes": 0, "answerable": 0, "unknown": 0,
"max_candidates": 0})
bucket["episodes"] += 1
bucket["answerable" if row["positive_indices"] else "unknown"] += 1
bucket["max_candidates"] = max(bucket["max_candidates"], len(row["candidates"]))
manifest = {
"generator": "make_realistic_memory_eval.py",
"purpose": "realistic-shaped memory evaluation corpus (multi-entity, aliases, updates, "
"multi-hop, near-miss, noise, long facts, unknowns)",
"per_category": args.per_category,
"seed": args.seed,
"episodes": len(episodes),
"sha256": hashlib.sha256(output.read_bytes()).hexdigest(),
"categories": counts,
"note": ("episodes use the schema consumed by eval_end_to_end_memory "
"(query / candidates[].text / positive_indices / metadata.acceptable)"),
}
(output.with_suffix(".manifest.json")).write_text(
json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps(manifest, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())