WpyQwq
643e22ecb9
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线 - 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同） - 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00） - 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00% - 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益） - 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
300 lines
16 KiB
Python
300 lines
16 KiB
Python
"""Build a *realistic-shaped* memory evaluation corpus.
|
||
|
||
Why: every number this project has produced so far rests on a template corpus -- one
|
||
sentence shape ("我的X是 VAL-…。"), a single entity (user), 24 attributes and no updates,
|
||
conflicts, multi-hop chains or noise. Those numbers are internally consistent and
|
||
reproducible, but none of them predict behaviour on text that looks like a real user's.
|
||
This generator produces the missing population.
|
||
|
||
Eight categories, each generating the same episode schema the existing harnesses consume
|
||
(``query`` / ``candidates[].text`` / ``positive_indices`` / ``metadata.acceptable``):
|
||
|
||
multi_entity 同一属性在多个实体上出现，问题只问其中一个
|
||
alias_paraphrase 提问与事实无字面重叠（换一种说法）
|
||
update_conflict 同一实体+属性写过两次，正确答案是较新那个，旧值是强干扰
|
||
multi_hop 答案需要串联两条事实（两条都是正例）
|
||
near_miss 候选属性名高度相近（手机尾号 / 座机尾号）
|
||
noise_context 库里混入大量无关闲聊，问题仍然可回答
|
||
long_fact 事实是长句，值嵌在句子中间
|
||
unknown_attribute 问一个从未写过的属性（不可回答，必须拒答）
|
||
|
||
Usage::
|
||
|
||
python -m V2_dpskw.make_realistic_memory_eval --output data/realistic_eval.jsonl
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import random
|
||
from pathlib import Path
|
||
|
||
#: entity -> (subject phrase used inside facts, alias list used by questions)
|
||
ENTITIES = [
|
||
("user", ["我", "我本人"], ["我", "我自己"]),
|
||
("zhangsan", ["张三", "我们组的张三"], ["张三", "我们组那位张三"]),
|
||
("liworks", ["李工", "运维的李工"], ["李工", "运维那位李工"]),
|
||
("proj_alpha", ["项目 Alpha", "Alpha 项目"], ["Alpha", "Alpha 项目"]),
|
||
("client_beta", ["客户 Beta", "Beta 客户"], ["Beta", "Beta 客户"]),
|
||
]
|
||
|
||
#: attribute -> (fact frames, question frames without lexical overlap)
|
||
ATTRIBUTES = [
|
||
("值班电话", ["{s}的值班电话是 {v}。", "{s}留的值班号码为 {v}。"],
|
||
["{q}有急事按哪个号找人？", "{q}轮班时该拨什么号码？"]),
|
||
("常驻机房", ["{s}常驻机房在 {v}。", "{s}平时待的机房是 {v}。"],
|
||
["{q}平时在哪间屋子干活？", "{q}固定在哪个位置办公？"]),
|
||
("发布窗口", ["{s}的发布窗口是 {v}。", "{s}固定 {v} 做上线。"],
|
||
["{q}一般什么时候做上线？", "{q}挑哪个时段变更？"]),
|
||
("接口版本号", ["{s}对接的接口版本号是 {v}。", "{s}目前跑的是 {v} 版本接口。"],
|
||
["{q}现在对接的是哪一版？", "{q}联调用的那个版本是什么？"]),
|
||
("告警阈值", ["{s}的告警阈值是 {v}。", "{s}超过 {v} 就报警。"],
|
||
["{q}到多少会触发报警？", "{q}哪条线一破就叫？"]),
|
||
("应急联系人", ["{s}的应急联系人是 {v}。", "{s}出事找 {v}。"],
|
||
["{q}出问题该喊谁？", "{q}紧急情况下找谁接手？"]),
|
||
("主库地址", ["{s}的主库地址是 {v}。", "{s}主库落在 {v}。"],
|
||
["{q}数据落在哪一个地址？", "{q}写库连的是哪里？"]),
|
||
("缓存容量", ["{s}的缓存容量是 {v}。", "{s}缓存开到 {v}。"],
|
||
["{q}内存开到多大？", "{q}缓冲配了多少？"]),
|
||
]
|
||
|
||
#: attribute pairs that are deliberately easy to confuse
|
||
NEAR_MISS = [
|
||
("手机尾号", "座机尾号"),
|
||
("办公城市", "常住城市"),
|
||
("工位楼层", "宿舍楼层"),
|
||
("项目代号", "客户代号"),
|
||
]
|
||
|
||
_VALUES = ["分机 8821", "B3-204", "每周三 22:00", "v3.14.2", "91.5%", "王工 138****6621",
|
||
"10.20.3.7:5432", "64GB", "A-7719", "夜间 02:00", "v2.8.0", "78%",
|
||
"李工 139****3344", "10.20.9.1:6379", "128GB", "B-3310"]
|
||
|
||
_ALIAS_FACTS = [
|
||
("我平时待得最久的那座城是 {v}。", "我常年落脚在哪儿？"),
|
||
("我每天睡醒最早看的那个数字是 {v}。", "我那个数字是多少？"),
|
||
("我手上那台机器是 {v}。", "我用的那台是什么款？"),
|
||
("我每天早上灌下去的那杯偏爱 {v}。", "我早上那杯是什么口味？"),
|
||
]
|
||
|
||
_QUESTION_POOL = ["{q}这个是什么？", "{q}对应的值是多少？"]
|
||
|
||
|
||
def _value(rng: random.Random) -> str:
|
||
return rng.choice(_VALUES)
|
||
|
||
|
||
def _episode(index: int, category: str, query: str, candidates: list[str],
|
||
positive_texts: list[str], acceptable: list[str], *, hop: int = 1,
|
||
extra: dict | None = None) -> dict:
|
||
positives = [position for position, text in enumerate(candidates) if text in positive_texts]
|
||
metadata = {
|
||
"category": category,
|
||
"answerable": bool(positives),
|
||
"acceptable": acceptable if positives else [],
|
||
"hop_count": hop if positives else 0,
|
||
"answer": acceptable[0] if acceptable and positives else "",
|
||
}
|
||
if extra:
|
||
metadata.update(extra)
|
||
return {
|
||
"id": "realistic-%s-%05d" % (category, index),
|
||
"group_id": "realistic-%s-%05d" % (category, index),
|
||
"source": "realistic_memory_eval",
|
||
"family": category,
|
||
"query": query,
|
||
"candidates": [{"text": text, "kind": "fact", "entity": "user", "attribute": ""}
|
||
for text in candidates],
|
||
"positive_indices": positives,
|
||
"positive_index": positives[0] if positives else -1,
|
||
"need_memory": 1.0 if positives else 0.0,
|
||
"hop": hop if positives else 0,
|
||
"metadata": metadata,
|
||
}
|
||
|
||
|
||
def build(count_per_category: int, seed: int) -> list[dict]:
|
||
rng = random.Random(seed)
|
||
episodes: list[dict] = []
|
||
counter = 0
|
||
|
||
for _ in range(count_per_category):
|
||
# --- multi_entity: same attribute on several entities, one is asked about -----
|
||
attribute, fact_frames, question_frames = rng.choice(ATTRIBUTES)
|
||
speakers = rng.sample(ENTITIES, 3)
|
||
target = rng.choice(speakers)
|
||
candidates, values = [], {}
|
||
for key, subjects, aliases in speakers:
|
||
value = _value(rng)
|
||
values[key] = value
|
||
candidates.append(rng.choice(fact_frames).format(s=subjects[0], v=value))
|
||
rng.shuffle(candidates)
|
||
fact = rng.choice(fact_frames).format(s=target[1][0], v=values[target[0]])
|
||
# Rebuild so the target's own phrasing is present verbatim among candidates.
|
||
candidates = []
|
||
for key, subjects, aliases in speakers:
|
||
frame = fact_frames[0] if key == target[0] else fact_frames[1 % len(fact_frames)]
|
||
candidates.append(frame.format(s=subjects[0], v=values[key]))
|
||
rng.shuffle(candidates)
|
||
fact = [c for c in candidates if values[target[0]] in c][0]
|
||
query = rng.choice(question_frames).format(q=rng.choice(target[2]))
|
||
episodes.append(_episode(counter, "multi_entity", query, candidates, [fact],
|
||
[values[target[0]]], extra={"attribute": attribute}))
|
||
counter += 1
|
||
|
||
# --- alias_paraphrase: fact and question share no distinctive characters -------
|
||
# The target fact is the ONLY fact written in its own frame; every distractor uses
|
||
# a different attribute's frame, so the question identifies exactly one value.
|
||
# (An earlier version wrote the same frame with five different values, which is not
|
||
# a paraphrase test at all but a self-contradictory bank.)
|
||
target_index = rng.randrange(len(ATTRIBUTES))
|
||
template, question = rng.choice(_ALIAS_FACTS)
|
||
value = _value(rng)
|
||
fact = template.format(v=value)
|
||
others = [ATTRIBUTES[i][1][0].format(s="我", v=_value(rng))
|
||
for i in range(len(ATTRIBUTES)) if i != target_index][:4]
|
||
candidates = [fact] + others
|
||
rng.shuffle(candidates)
|
||
episodes.append(_episode(counter, "alias_paraphrase", question, candidates, [fact], [value]))
|
||
counter += 1
|
||
|
||
# --- update_conflict: older value is the strong distractor ---------------------
|
||
attribute, fact_frames, question_frames = rng.choice(ATTRIBUTES)
|
||
old_value, new_value = _value(rng), _value(rng)
|
||
while new_value == old_value:
|
||
new_value = _value(rng)
|
||
old_fact = fact_frames[0].format(s="我", v=old_value)
|
||
new_fact = fact_frames[1 % len(fact_frames)].format(s="我", v=new_value)
|
||
candidates = [old_fact, new_fact]
|
||
candidates += [fact_frames[0].format(s="我", v=_value(rng)) for _ in range(3)]
|
||
rng.shuffle(candidates)
|
||
query = rng.choice(question_frames).format(q="我")
|
||
episodes.append(_episode(counter, "update_conflict", query, candidates, [new_fact],
|
||
[new_value], extra={"attribute": attribute,
|
||
"superseded_value": old_value}))
|
||
counter += 1
|
||
|
||
# --- multi_hop: the answer needs two chained facts, and is a *unique* token ------
|
||
# The answer must not be recoverable by blending: every chain points at a distinct
|
||
# code, so only following the right chain yields the expected token.
|
||
code = "REF-%s" % "".join(rng.choice("ABCDEFGHJKLMNPQRSTUVWXYZ23456789") for _ in range(6))
|
||
answer = "值班人-7741"
|
||
others = ["值班人-%04d" % rng.randrange(1000, 9999) for _ in range(3)]
|
||
first = "张三负责的那个项目的内部编号是 %s。" % code
|
||
second = "编号 %s 对应的值班人是 %s。" % (code, answer)
|
||
decoy_codes = ["REF-" + "".join(rng.choice("ABCDEFGHJKLMNPQRSTUVWXYZ23456789")
|
||
for _ in range(6)) for _ in others]
|
||
decoys = ["编号 %s 对应的值班人是 %s。" % (dc, name)
|
||
for dc, name in zip(decoy_codes, others)]
|
||
candidates = [first, second] + decoys
|
||
rng.shuffle(candidates)
|
||
query = "负责那个项目的同事对应的值班人是谁？"
|
||
episodes.append(_episode(counter, "multi_hop", query, candidates, [first, second],
|
||
[answer], hop=2))
|
||
counter += 1
|
||
|
||
# --- near_miss: the question refers to the attribute without naming it ----------
|
||
# Naming the attribute outright ("我的手机尾号是什么") is not a hard test; the
|
||
# question here paraphrases it so the near-miss decoy is genuinely in play.
|
||
left, right = rng.choice(NEAR_MISS)
|
||
value, decoy = _value(rng), _value(rng)
|
||
candidates = ["我的%s是 %s。" % (left, value), "我的%s是 %s。" % (right, decoy)]
|
||
candidates += ["我的%s是 %s。" % (name, _value(rng))
|
||
for name, _, _ in rng.sample(ATTRIBUTES, 2)]
|
||
rng.shuffle(candidates)
|
||
ask_left = rng.random() < 0.5
|
||
target_value = value if ask_left else decoy
|
||
target_fact = [c for c in candidates if target_value in c][0]
|
||
query = ("我随身那台设备的末几位数字是啥？" if ask_left
|
||
else "我家座机最后几位数字是啥？")
|
||
episodes.append(_episode(counter, "near_miss", query, candidates, [target_fact],
|
||
[target_value], extra={"attribute": left if ask_left else right}))
|
||
counter += 1
|
||
|
||
# --- noise_context: lots of unrelated chatter, question still answerable -------
|
||
attribute, fact_frames, question_frames = rng.choice(ATTRIBUTES)
|
||
value = _value(rng)
|
||
fact = fact_frames[0].format(s="我", v=value)
|
||
noise = ["这是普通对话噪声：验证用户-%06d 暂时提到一个无关编号 N-%s，不需要长期保存。"
|
||
% (rng.randrange(10 ** 6), "".join(rng.choice("ABCDEFGHJKLMNPQRSTUVWXYZ23456789")
|
||
for _ in range(8)))
|
||
for _ in range(8)]
|
||
candidates = [fact] + noise
|
||
rng.shuffle(candidates)
|
||
query = rng.choice(question_frames).format(q="我")
|
||
episodes.append(_episode(counter, "noise_context", query, candidates, [fact], [value]))
|
||
counter += 1
|
||
|
||
# --- long_fact: the value sits mid-sentence in a long fact ---------------------
|
||
attribute, fact_frames, question_frames = rng.choice(ATTRIBUTES)
|
||
value = _value(rng)
|
||
long_fact = ("关于%s这件事，之前散会时我们临时定了下来，当时讨论得比较久，最后确认%s，"
|
||
"后续如果还有调整会在群里同步，暂时先按这个执行。" % (attribute, value))
|
||
others = [("关于%s这件事，会上也提过一嘴，但那次的说法是%s，后来没有正式确认过。"
|
||
% (attribute, _value(rng))) for _ in range(3)]
|
||
candidates = [long_fact] + others
|
||
rng.shuffle(candidates)
|
||
query = rng.choice(question_frames).format(q="我")
|
||
episodes.append(_episode(counter, "long_fact", query, candidates, [long_fact], [value]))
|
||
counter += 1
|
||
|
||
# --- unknown_attribute: the asked attribute was never written ------------------
|
||
# Candidates must be written in *their own* frames (an earlier version reused the
|
||
# asked attribute's frame, so the "unknown" attribute was in fact present and the
|
||
# question was answerable -- the 0.00% it produced measured nothing).
|
||
attribute, fact_frames, question_frames = rng.choice(ATTRIBUTES)
|
||
written = [(name, frames) for name, frames, _ in ATTRIBUTES if name != attribute]
|
||
picked = rng.sample(written, 5)
|
||
candidates = [frames[0].format(s="我", v=_value(rng)) for _, frames in picked]
|
||
query = rng.choice(question_frames).format(q="我")
|
||
episodes.append(_episode(counter, "unknown_attribute", query, candidates, [], []))
|
||
counter += 1
|
||
|
||
return episodes
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument("--output", default="data/realistic_eval.jsonl")
|
||
parser.add_argument("--per-category", type=int, default=25)
|
||
parser.add_argument("--seed", type=int, default=20260913)
|
||
args = parser.parse_args()
|
||
|
||
episodes = build(args.per_category, args.seed)
|
||
output = Path(args.output)
|
||
output.parent.mkdir(parents=True, exist_ok=True)
|
||
with output.open("w", encoding="utf-8") as handle:
|
||
for row in episodes:
|
||
handle.write(json.dumps(row, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||
|
||
counts: dict[str, dict] = {}
|
||
for row in episodes:
|
||
category = row["metadata"]["category"]
|
||
bucket = counts.setdefault(category, {"episodes": 0, "answerable": 0, "unknown": 0,
|
||
"max_candidates": 0})
|
||
bucket["episodes"] += 1
|
||
bucket["answerable" if row["positive_indices"] else "unknown"] += 1
|
||
bucket["max_candidates"] = max(bucket["max_candidates"], len(row["candidates"]))
|
||
manifest = {
|
||
"generator": "make_realistic_memory_eval.py",
|
||
"purpose": "realistic-shaped memory evaluation corpus (multi-entity, aliases, updates, "
|
||
"multi-hop, near-miss, noise, long facts, unknowns)",
|
||
"per_category": args.per_category,
|
||
"seed": args.seed,
|
||
"episodes": len(episodes),
|
||
"sha256": hashlib.sha256(output.read_bytes()).hexdigest(),
|
||
"categories": counts,
|
||
"note": ("episodes use the schema consumed by eval_end_to_end_memory "
|
||
"(query / candidates[].text / positive_indices / metadata.acceptable)"),
|
||
}
|
||
(output.with_suffix(".manifest.json")).write_text(
|
||
json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
print(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|