Natural Memory NM2.1: 记忆路由器分叉、数据集缺陷修复与全轴评测证据
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线 - 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同） - 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00） - 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00% - 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益） - 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
This commit is contained in:
WpyQwq
@@ -0,0 +1,314 @@
|
||||
{
|
||||
"scorers": {
|
||||
"cosine": {
|
||||
"single_feature_auc_on_eval": {
|
||||
"top1": 0.5898,
|
||||
"top2": 0.5572,
|
||||
"margin": 0.6257,
|
||||
"top3_mean": 0.5771,
|
||||
"top1_minus_median": 0.6166,
|
||||
"std": 0.5894,
|
||||
"z_top1": 0.6125,
|
||||
"n_within_10pct": 0.4273,
|
||||
"entropy": 0.6168
|
||||
},
|
||||
"head_auc_on_eval": 0.6086,
|
||||
"head_weights": {
|
||||
"top1": 0.2654,
|
||||
"top2": 0.3455,
|
||||
"margin": -0.2102,
|
||||
"top3_mean": -0.4675,
|
||||
"top1_minus_median": -0.0127,
|
||||
"std": -0.1578,
|
||||
"z_top1": -0.2808,
|
||||
"n_within_10pct": 0.1283,
|
||||
"entropy": 0.9424
|
||||
},
|
||||
"operating_curve": [
|
||||
{
|
||||
"threshold": 0.05,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.1,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.2,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.3,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.4,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.5,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.6,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.7,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.8,
|
||||
"unknown_flagged_pct": 46.0,
|
||||
"answerable_rejected_pct": 30.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.9,
|
||||
"unknown_flagged_pct": 96.0,
|
||||
"answerable_rejected_pct": 84.0
|
||||
}
|
||||
]
|
||||
},
|
||||
"router": {
|
||||
"single_feature_auc_on_eval": {
|
||||
"top1": 0.5379,
|
||||
"top2": 0.5263,
|
||||
"margin": 0.5516,
|
||||
"top3_mean": 0.5386,
|
||||
"top1_minus_median": 0.5108,
|
||||
"std": 0.5447,
|
||||
"z_top1": 0.5711,
|
||||
"n_within_10pct": 0.426,
|
||||
"entropy": 0.497
|
||||
},
|
||||
"head_auc_on_eval": 0.513,
|
||||
"head_weights": {
|
||||
"top1": 0.259,
|
||||
"top2": 0.0524,
|
||||
"margin": 0.3174,
|
||||
"top3_mean": 0.0438,
|
||||
"top1_minus_median": 0.0431,
|
||||
"std": 0.0405,
|
||||
"z_top1": 0.4694,
|
||||
"n_within_10pct": 0.4964,
|
||||
"entropy": 0.0123
|
||||
},
|
||||
"operating_curve": [
|
||||
{
|
||||
"threshold": 0.05,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.1,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.2,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.3,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.4,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.5,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.6,
|
||||
"unknown_flagged_pct": 4.0,
|
||||
"answerable_rejected_pct": 5.2
|
||||
},
|
||||
{
|
||||
"threshold": 0.7,
|
||||
"unknown_flagged_pct": 4.0,
|
||||
"answerable_rejected_pct": 7.2
|
||||
},
|
||||
{
|
||||
"threshold": 0.8,
|
||||
"unknown_flagged_pct": 38.0,
|
||||
"answerable_rejected_pct": 34.8
|
||||
},
|
||||
{
|
||||
"threshold": 0.9,
|
||||
"unknown_flagged_pct": 90.0,
|
||||
"answerable_rejected_pct": 94.0
|
||||
}
|
||||
]
|
||||
},
|
||||
"text_retriever": {
|
||||
"single_feature_auc_on_eval": {
|
||||
"top1": 0.4992,
|
||||
"top2": 0.4966,
|
||||
"margin": 0.6076,
|
||||
"top3_mean": 0.4954,
|
||||
"top1_minus_median": 0.5712,
|
||||
"std": 0.5043,
|
||||
"z_top1": 0.5512,
|
||||
"n_within_10pct": 0.4462,
|
||||
"entropy": 0.5655
|
||||
},
|
||||
"head_auc_on_eval": 0.4752,
|
||||
"head_weights": {
|
||||
"top1": -0.3237,
|
||||
"top2": -0.3446,
|
||||
"margin": 0.0748,
|
||||
"top3_mean": 0.8565,
|
||||
"top1_minus_median": 0.0869,
|
||||
"std": 0.4047,
|
||||
"z_top1": 0.1123,
|
||||
"n_within_10pct": -0.2813,
|
||||
"entropy": -0.4289
|
||||
},
|
||||
"operating_curve": [
|
||||
{
|
||||
"threshold": 0.05,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.1,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.2,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.3,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.4,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.5,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.6,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.7,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 1.2
|
||||
},
|
||||
{
|
||||
"threshold": 0.8,
|
||||
"unknown_flagged_pct": 14.0,
|
||||
"answerable_rejected_pct": 15.2
|
||||
},
|
||||
{
|
||||
"threshold": 0.9,
|
||||
"unknown_flagged_pct": 100.0,
|
||||
"answerable_rejected_pct": 100.0
|
||||
}
|
||||
]
|
||||
},
|
||||
"blend_50_50": {
|
||||
"single_feature_auc_on_eval": {
|
||||
"top1": 0.4788,
|
||||
"top2": 0.493,
|
||||
"margin": 0.5446,
|
||||
"top3_mean": 0.4929,
|
||||
"top1_minus_median": 0.4943,
|
||||
"std": 0.5042,
|
||||
"z_top1": 0.5132,
|
||||
"n_within_10pct": 0.5104,
|
||||
"entropy": 0.4914
|
||||
},
|
||||
"head_auc_on_eval": 0.5062,
|
||||
"head_weights": {
|
||||
"top1": 0.2989,
|
||||
"top2": 0.3659,
|
||||
"margin": -0.0757,
|
||||
"top3_mean": -0.2102,
|
||||
"top1_minus_median": -0.1659,
|
||||
"std": 0.3382,
|
||||
"z_top1": 0.4046,
|
||||
"n_within_10pct": -0.0748,
|
||||
"entropy": -0.0503
|
||||
},
|
||||
"operating_curve": [
|
||||
{
|
||||
"threshold": 0.05,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.1,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.2,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.3,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.4,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.5,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.6,
|
||||
"unknown_flagged_pct": 0.0,
|
||||
"answerable_rejected_pct": 0.0
|
||||
},
|
||||
{
|
||||
"threshold": 0.7,
|
||||
"unknown_flagged_pct": 2.0,
|
||||
"answerable_rejected_pct": 0.4
|
||||
},
|
||||
{
|
||||
"threshold": 0.8,
|
||||
"unknown_flagged_pct": 22.0,
|
||||
"answerable_rejected_pct": 28.4
|
||||
},
|
||||
{
|
||||
"threshold": 0.9,
|
||||
"unknown_flagged_pct": 72.0,
|
||||
"answerable_rejected_pct": 74.0
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"train_episodes": 1200,
|
||||
"eval_episodes": 300
|
||||
}
|
||||
Reference in New Issue
Block a user