Natural Memory NM2.1: 记忆路由器分叉、数据集缺陷修复与全轴评测证据
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线 - 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同） - 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00） - 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00% - 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益） - 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
This commit is contained in:
WpyQwq
@@ -0,0 +1,186 @@
|
||||
{
|
||||
"format_version": 3,
|
||||
"generator": "prepare_memory_router_dataset.py",
|
||||
"seed": 20260909,
|
||||
"candidate_count": 32,
|
||||
"schema": {
|
||||
"query": "natural-language routing query",
|
||||
"candidates": "bounded memory records with text and provenance",
|
||||
"positive_indices": "one or more supporting memory records; empty means abstain",
|
||||
"need_memory": "1 if evidence is required and present, 0 for unknown/unsupported queries",
|
||||
"hop": "0 for abstention, otherwise number of supporting records clipped to router max_hops",
|
||||
"group_id": "conversation or QA episode identity; no group may cross train/eval"
|
||||
},
|
||||
"files": {
|
||||
"train": {
|
||||
"path": "H:\\Memory\\V2_dpskw\\data\\_probe_original\\train.jsonl",
|
||||
"episodes": 300,
|
||||
"sha256": "433df3400497202828aab3a7857151e8191052c5e1d6d2bd03bbde4de7d97e26"
|
||||
},
|
||||
"eval": {
|
||||
"path": "H:\\Memory\\V2_dpskw\\data\\_probe_original\\eval.jsonl",
|
||||
"episodes": 100,
|
||||
"sha256": "9e62a5e24cf27828a4dfb011e2f5b86975353eb2a373a8ed65f1133ee143e032"
|
||||
}
|
||||
},
|
||||
"counts": {
|
||||
"train_groups": 84155,
|
||||
"eval_groups": 21120,
|
||||
"train_episodes": 300,
|
||||
"eval_episodes": 100,
|
||||
"train_unknown": 162,
|
||||
"eval_unknown": 73,
|
||||
"train_candidates": 9600,
|
||||
"eval_candidates": 3200
|
||||
},
|
||||
"local_sources": [
|
||||
{
|
||||
"source": "dynamic_memory_lab/data/benchmark_train.jsonl",
|
||||
"rows": 128,
|
||||
"kind": "benchmark",
|
||||
"split": "train"
|
||||
},
|
||||
{
|
||||
"source": "dynamic_memory_lab/data/native_memory/train.jsonl",
|
||||
"rows": 512,
|
||||
"kind": "native_memory",
|
||||
"split": "train"
|
||||
},
|
||||
{
|
||||
"source": "dynamic_memory_lab/data/production_memory/train.jsonl",
|
||||
"rows": 1485,
|
||||
"kind": "normalized_policy",
|
||||
"split": "train"
|
||||
},
|
||||
{
|
||||
"source": "dynamic_memory_lab/data/production_memory_hard_v2/train.jsonl",
|
||||
"rows": 27000,
|
||||
"kind": "normalized_policy",
|
||||
"split": "train"
|
||||
},
|
||||
{
|
||||
"source": "dynamic_memory_lab/data/mega_validation/memory_validation_80k_train.jsonl",
|
||||
"rows": 80000,
|
||||
"kind": "mega_validation",
|
||||
"split": "train"
|
||||
},
|
||||
{
|
||||
"source": "dynamic_memory_lab/data/benchmark_eval.jsonl",
|
||||
"rows": 64,
|
||||
"kind": "benchmark",
|
||||
"split": "eval"
|
||||
},
|
||||
{
|
||||
"source": "dynamic_memory_lab/data/native_memory/eval.jsonl",
|
||||
"rows": 128,
|
||||
"kind": "native_memory",
|
||||
"split": "eval"
|
||||
},
|
||||
{
|
||||
"source": "dynamic_memory_lab/data/production_memory/eval.jsonl",
|
||||
"rows": 363,
|
||||
"kind": "normalized_policy",
|
||||
"split": "eval"
|
||||
},
|
||||
{
|
||||
"source": "dynamic_memory_lab/data/production_memory_hard_v2/eval.jsonl",
|
||||
"rows": 7200,
|
||||
"kind": "normalized_policy",
|
||||
"split": "eval"
|
||||
},
|
||||
{
|
||||
"source": "dynamic_memory_lab/data/mega_validation/memory_validation_20k_eval.jsonl",
|
||||
"rows": 20000,
|
||||
"kind": "mega_validation",
|
||||
"split": "eval"
|
||||
}
|
||||
],
|
||||
"public_sources": [],
|
||||
"public_catalog": [
|
||||
{
|
||||
"name": "hotpotqa_train",
|
||||
"dataset_id": "hotpot_qa",
|
||||
"config": "distractor",
|
||||
"split": "train",
|
||||
"split_kind": "train",
|
||||
"task": "qa",
|
||||
"url": "https://huggingface.co/datasets/hotpot_qa"
|
||||
},
|
||||
{
|
||||
"name": "hotpotqa_validation",
|
||||
"dataset_id": "hotpot_qa",
|
||||
"config": "distractor",
|
||||
"split": "validation",
|
||||
"split_kind": "eval",
|
||||
"task": "qa",
|
||||
"url": "https://huggingface.co/datasets/hotpot_qa"
|
||||
},
|
||||
{
|
||||
"name": "codesearchnet_python_train",
|
||||
"dataset_id": "code_search_net",
|
||||
"config": "python",
|
||||
"split": "train",
|
||||
"split_kind": "train",
|
||||
"task": "code",
|
||||
"url": "https://huggingface.co/datasets/code_search_net"
|
||||
},
|
||||
{
|
||||
"name": "codesearchnet_python_validation",
|
||||
"dataset_id": "code_search_net",
|
||||
"config": "python",
|
||||
"split": "validation",
|
||||
"split_kind": "eval",
|
||||
"task": "code",
|
||||
"url": "https://huggingface.co/datasets/code_search_net"
|
||||
},
|
||||
{
|
||||
"name": "fever_train",
|
||||
"dataset_id": "fever",
|
||||
"config": "v1.0",
|
||||
"split": "train",
|
||||
"split_kind": "train",
|
||||
"task": "evidence",
|
||||
"url": "https://huggingface.co/datasets/fever"
|
||||
},
|
||||
{
|
||||
"name": "fever_validation",
|
||||
"dataset_id": "fever",
|
||||
"config": "v1.0",
|
||||
"split": "labelled_dev",
|
||||
"split_kind": "eval",
|
||||
"task": "evidence",
|
||||
"url": "https://huggingface.co/datasets/fever"
|
||||
}
|
||||
],
|
||||
"split_stats": {
|
||||
"train": {
|
||||
"kept": 300,
|
||||
"unknown": 162,
|
||||
"family:benchmark_qa": 128,
|
||||
"family:native_memory": 172,
|
||||
"written": 300
|
||||
},
|
||||
"eval": {
|
||||
"kept": 100,
|
||||
"unknown": 73,
|
||||
"family:benchmark_qa": 64,
|
||||
"family:native_memory": 36,
|
||||
"written": 100
|
||||
}
|
||||
},
|
||||
"leakage_check": {
|
||||
"group_overlap": 0,
|
||||
"passed": true
|
||||
},
|
||||
"conflict_policy": {
|
||||
"enabled": true,
|
||||
"answerable_positive_conflicts_excluded": true,
|
||||
"unknown_hard_conflicts_retained": true
|
||||
},
|
||||
"evaluation_policy": "eval.jsonl is generated and hashed before router training; the trainer refuses an optional hash mismatch.",
|
||||
"warnings": [
|
||||
"Local generated memory-policy files are useful hard negatives but are not public-human chat data.",
|
||||
"Add redacted real user traces only after consent and PII removal.",
|
||||
"Public download failures are recorded; failed sources contribute zero rows."
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user