wpy/natural-memory-nm21
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
643e22ecb905392eedef8b52ce7c9d0c7f185ed1
natural-memory-nm21/_regress_scorecard.json
T
WpyQwq 643e22ecb9 Natural Memory NM2.1: 记忆路由器分叉、数据集缺陷修复与全轴评测证据 
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线
- 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同）
- 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00）
- 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00%
- 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益）
- 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
2026-09-19 11:11:31 +08:00

2456 lines
78 KiB
JSON
Raw Blame History

{
"V2-512 best(step1000)": {
"checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_router_512\\router_best.pt",
"router": {
"kind": "router_v2",
"step": 1000,
"arch_config": {
"arch": "router_v2",
"hidden_size": 2560,
"router_dim": 512,
"num_heads": 8,
"max_hops": 3
},
"parameters": 4741902,
"checkpoint_bytes": 56922381
},
"storage": {
"address_bytes_per_record": 2048,
"address_mb_per_1m_records": 2048.0,
"checkpoint_bytes": 56922381
},
"latency": {
"cpu": {
"single_query_latency_ms_mean": 1.2766800005920231,
"single_query_latency_ms_p50": 1.2820999836549163,
"single_query_latency_ms_p95": 1.3297999976202846,
"queries_per_second": 783.2816363820841,
"samples": 5
}
},
"metrics": {
"episodes": 1940,
"answerable_episodes": 1834,
"unknown_episodes": 106,
"top1": 0.6712104689203926,
"recall1": 0.6712104689203926,
"recall3": 0.8876772082878953,
"recall5": 0.9492911668484187,
"recall8": 0.9858233369683751,
"mrr": 0.7908385671852008,
"ndcg3": 0.7999554748364778,
"mean_first_positive_rank": 1.841875681570338,
"all_evidence_in_top3": 0.8865866957470011,
"multi_positive_episodes": 2,
"all_evidence_in_top3_multi": 0.5,
"hop_accuracy": 0.9783505154639175,
"hop_accuracy_answerable": 0.9967284623773174,
"hop_under_prediction": 0.003271537622682661,
"hop_over_prediction": 0.0,
"mean_score_margin": 0.5880087030850424,
"median_score_margin": 0.347680926322937,
"negative_margin_rate": 0.3287895310796074,
"hop_confusion": {
"true_0_pred_0": 70,
"true_0_pred_1": 36,
"true_0_pred_2": 0,
"true_1_pred_0": 4,
"true_1_pred_1": 1828,
"true_1_pred_2": 0,
"true_2_pred_0": 0,
"true_2_pred_1": 2,
"true_2_pred_2": 0
},
"known_mean_need_prob": 0.9847397789147844,
"unknown_mean_need_prob": 0.2536155520674755,
"thr0.30": {
"need_precision": 0.9786096256684492,
"need_recall": 0.9978189749182116,
"need_f1": 0.9881209503239742,
"specificity_unknown_refusal": 0.6226415094339622,
"abstention_accuracy": 0.977319587628866,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.37735849056603776,
"counts": {
"tp": 1830,
"tn": 66,
"fp": 40,
"fn": 4
}
},
"thr0.40": {
"need_precision": 0.9796573875802997,
"need_recall": 0.9978189749182116,
"need_f1": 0.9886547811993517,
"specificity_unknown_refusal": 0.6415094339622641,
"abstention_accuracy": 0.9783505154639175,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.3584905660377358,
"counts": {
"tp": 1830,
"tn": 68,
"fp": 38,
"fn": 4
}
},
"thr0.50": {
"need_precision": 0.9807073954983923,
"need_recall": 0.9978189749182116,
"need_f1": 0.9891891891891892,
"specificity_unknown_refusal": 0.660377358490566,
"abstention_accuracy": 0.979381443298969,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.33962264150943394,
"counts": {
"tp": 1830,
"tn": 70,
"fp": 36,
"fn": 4
}
},
"thr0.60": {
"need_precision": 0.9902597402597403,
"need_recall": 0.9978189749182116,
"need_f1": 0.9940249864204237,
"specificity_unknown_refusal": 0.8301886792452831,
"abstention_accuracy": 0.988659793814433,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.16981132075471697,
"counts": {
"tp": 1830,
"tn": 88,
"fp": 18,
"fn": 4
}
},
"thr0.70": {
"need_precision": 0.9989082969432315,
"need_recall": 0.9978189749182116,
"need_f1": 0.9983633387888707,
"specificity_unknown_refusal": 0.9811320754716981,
"abstention_accuracy": 0.9969072164948454,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.018867924528301886,
"counts": {
"tp": 1830,
"tn": 104,
"fp": 2,
"fn": 4
}
}
},
"by_family": {
"benchmark_qa": {
"episodes": 64,
"answerable_episodes": 0,
"unknown_episodes": 64,
"top1": null,
"recall1": null,
"recall3": null,
"recall5": null,
"recall8": null,
"mrr": null,
"ndcg3": null,
"mean_first_positive_rank": null,
"all_evidence_in_top3": null,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": null,
"hop_under_prediction": null,
"hop_over_prediction": null,
"mean_score_margin": null,
"median_score_margin": null,
"negative_margin_rate": null,
"hop_confusion": {
"true_0_pred_0": 64
},
"known_mean_need_prob": null,
"unknown_mean_need_prob": 0.04612289386568591,
"thr0.50": {
"need_precision": null,
"need_recall": null,
"need_f1": null,
"specificity_unknown_refusal": 1.0,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": null,
"unknown_question_read_rate": 0.0,
"counts": {
"tp": 0,
"tn": 64,
"fp": 0,
"fn": 0
}
}
},
"mega_validation": {
"episodes": 20,
"answerable_episodes": 20,
"unknown_episodes": 0,
"top1": 0.4,
"recall1": 0.4,
"recall3": 0.8,
"recall5": 1.0,
"recall8": 1.05,
"mrr": 0.6225574712643678,
"ndcg3": 0.6086215664331643,
"mean_first_positive_rank": 3.45,
"all_evidence_in_top3": 0.7,
"multi_positive_episodes": 2,
"all_evidence_in_top3_multi": 0.5,
"hop_accuracy": 0.7,
"hop_accuracy_answerable": 0.7,
"hop_under_prediction": 0.3,
"hop_over_prediction": 0.0,
"mean_score_margin": -0.29634838700294497,
"median_score_margin": -0.25011515617370605,
"negative_margin_rate": 0.6,
"hop_confusion": {
"true_1_pred_0": 4,
"true_1_pred_1": 14,
"true_1_pred_2": 0,
"true_2_pred_0": 0,
"true_2_pred_1": 2,
"true_2_pred_2": 0
},
"known_mean_need_prob": 0.8251703515648842,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 0.8,
"need_f1": 0.888888888888889,
"specificity_unknown_refusal": null,
"abstention_accuracy": 0.8,
"known_question_refusal_rate": 0.2,
"unknown_question_read_rate": null,
"counts": {
"tp": 16,
"tn": 0,
"fp": 0,
"fn": 4
}
}
},
"memory_policy": {
"episodes": 1728,
"answerable_episodes": 1707,
"unknown_episodes": 21,
"top1": 0.6988869361452841,
"recall1": 0.6988869361452841,
"recall3": 0.9004100761570006,
"recall5": 0.9537199765670767,
"recall8": 0.9859402460456942,
"mrr": 0.8099543326670381,
"ndcg3": 0.8196676424617168,
"mean_first_positive_rank": 1.7492677211482133,
"all_evidence_in_top3": 0.9004100761570006,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 0.9895833333333334,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 0.6516404748137541,
"median_score_margin": 0.38489866256713867,
"negative_margin_rate": 0.30111306385471587,
"hop_confusion": {
"true_0_pred_0": 3,
"true_0_pred_1": 18,
"true_1_pred_0": 0,
"true_1_pred_1": 1707
},
"known_mean_need_prob": 0.9862654947298761,
"unknown_mean_need_prob": 0.5697948407559168,
"thr0.50": {
"need_precision": 0.9895652173913043,
"need_recall": 1.0,
"need_f1": 0.9947552447552448,
"specificity_unknown_refusal": 0.14285714285714285,
"abstention_accuracy": 0.9895833333333334,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": 0.8571428571428571,
"counts": {
"tp": 1707,
"tn": 3,
"fp": 18,
"fn": 0
}
}
},
"native_memory": {
"episodes": 128,
"answerable_episodes": 107,
"unknown_episodes": 21,
"top1": 0.2803738317757009,
"recall1": 0.2803738317757009,
"recall3": 0.7009345794392523,
"recall5": 0.8691588785046729,
"recall8": 0.9719626168224299,
"mrr": 0.5173339899975414,
"ndcg3": 0.5212452695260414,
"mean_first_positive_rank": 3.0186915887850465,
"all_evidence_in_top3": 0.7009345794392523,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 0.859375,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -0.26182580662664967,
"median_score_margin": -0.27979564666748047,
"negative_margin_rate": 0.719626168224299,
"hop_confusion": {
"true_0_pred_0": 3,
"true_0_pred_1": 18,
"true_1_pred_0": 0,
"true_1_pred_1": 107
},
"known_mean_need_prob": 0.9902256821917597,
"unknown_mean_need_prob": 0.5697948407559168,
"thr0.50": {
"need_precision": 0.856,
"need_recall": 1.0,
"need_f1": 0.9224137931034483,
"specificity_unknown_refusal": 0.14285714285714285,
"abstention_accuracy": 0.859375,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": 0.8571428571428571,
"counts": {
"tp": 107,
"tn": 3,
"fp": 18,
"fn": 0
}
}
}
},
"by_category": {
"conflict_update": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.75,
"ndcg3": 0.8154648767857288,
"mean_first_positive_rank": 1.5,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 0.25075769424438477,
"median_score_margin": 0.25075769424438477,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.9954056441783905,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"distractor_128": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 1.0,
"recall1": 1.0,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 1.0,
"ndcg3": 1.0,
"mean_first_positive_rank": 1.0,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 3.3609426021575928,
"median_score_margin": 3.3609426021575928,
"negative_margin_rate": 0.0,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.971575915813446,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"distractor_32": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.75,
"ndcg3": 0.8154648767857288,
"mean_first_positive_rank": 1.5,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 1.208745002746582,
"median_score_margin": 1.208745002746582,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.92432501912117,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"forget_correction": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 1.5,
"recall5": 2.0,
"recall8": 2.0,
"mrr": 0.75,
"ndcg3": 0.6934264036172708,
"mean_first_positive_rank": 1.5,
"all_evidence_in_top3": 0.5,
"multi_positive_episodes": 2,
"all_evidence_in_top3_multi": 0.5,
"hop_accuracy": 0.0,
"hop_accuracy_answerable": 0.0,
"hop_under_prediction": 1.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 10.865452527999878,
"median_score_margin": 10.865452527999878,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_2_pred_1": 2,
"true_2_pred_2": 0
},
"known_mean_need_prob": 0.9887627363204956,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"long_context": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 0.5,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.625,
"ndcg3": 0.5,
"mean_first_positive_rank": 2.5,
"all_evidence_in_top3": 0.5,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -1.6004953384399414,
"median_score_margin": -1.6004953384399414,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.9807772934436798,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"multi_hop": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.0,
"recall1": 0.0,
"recall3": 0.5,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.375,
"ndcg3": 0.31546487678572877,
"mean_first_positive_rank": 3.0,
"all_evidence_in_top3": 0.5,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 0.0,
"hop_accuracy_answerable": 0.0,
"hop_under_prediction": 1.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -3.4204822182655334,
"median_score_margin": -3.4204822182655334,
"negative_margin_rate": 1.0,
"hop_confusion": {
"true_1_pred_0": 2,
"true_1_pred_1": 0
},
"known_mean_need_prob": 0.1579926311969757,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": null,
"need_recall": 0.0,
"need_f1": null,
"specificity_unknown_refusal": null,
"abstention_accuracy": 0.0,
"known_question_refusal_rate": 1.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 0,
"tn": 0,
"fp": 0,
"fn": 2
}
}
},
"paraphrase": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.0,
"recall1": 0.0,
"recall3": 0.5,
"recall5": 0.5,
"recall8": 1.0,
"mrr": 0.3333333333333333,
"ndcg3": 0.31546487678572877,
"mean_first_positive_rank": 4.0,
"all_evidence_in_top3": 0.5,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -2.652513027191162,
"median_score_margin": -2.652513027191162,
"negative_margin_rate": 1.0,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.99253049492836,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"random_position": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 1.0,
"recall1": 1.0,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 1.0,
"ndcg3": 1.0,
"mean_first_positive_rank": 1.0,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 3.1475493907928467,
"median_score_margin": 3.1475493907928467,
"negative_margin_rate": 0.0,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.9879558682441711,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"single_fact": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.0,
"recall1": 0.0,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.5,
"ndcg3": 0.6309297535714575,
"mean_first_positive_rank": 2.0,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -0.12020683288574219,
"median_score_margin": -0.12020683288574219,
"negative_margin_rate": 1.0,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.9997169077396393,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"unknown": {
"episodes": 1920,
"answerable_episodes": 1814,
"unknown_episodes": 106,
"top1": 0.6742006615214995,
"recall1": 0.6742006615214995,
"recall3": 0.8886438809261301,
"recall5": 0.9487320837927232,
"recall8": 0.9851157662624035,
"mrr": 0.792693926566908,
"ndcg3": 0.8020649997361834,
"mean_first_positive_rank": 1.8241455347298787,
"all_evidence_in_top3": 0.8886438809261301,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 0.98125,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 0.5977590568897612,
"median_score_margin": 0.3488118648529053,
"negative_margin_rate": 0.32579933847850057,
"hop_confusion": {
"true_0_pred_0": 70,
"true_0_pred_1": 36,
"true_1_pred_0": 0,
"true_1_pred_1": 1814
},
"known_mean_need_prob": 0.9864990890288957,
"unknown_mean_need_prob": 0.2536155520674755,
"thr0.50": {
"need_precision": 0.9805405405405405,
"need_recall": 1.0,
"need_f1": 0.990174672489083,
"specificity_unknown_refusal": 0.660377358490566,
"abstention_accuracy": 0.98125,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": 0.33962264150943394,
"counts": {
"tp": 1814,
"tn": 70,
"fp": 36,
"fn": 0
}
}
},
"unknown_abstention": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.0,
"recall1": 0.0,
"recall3": 0.0,
"recall5": 0.5,
"recall8": 0.5,
"mrr": 0.14224137931034483,
"ndcg3": 0.0,
"mean_first_positive_rank": 16.5,
"all_evidence_in_top3": 0.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 0.0,
"hop_accuracy_answerable": 0.0,
"hop_under_prediction": 1.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -14.003233671188354,
"median_score_margin": -14.003233671188354,
"negative_margin_rate": 1.0,
"hop_confusion": {
"true_1_pred_0": 2,
"true_1_pred_1": 0
},
"known_mean_need_prob": 0.25266100466251373,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": null,
"need_recall": 0.0,
"need_f1": null,
"specificity_unknown_refusal": null,
"abstention_accuracy": 0.0,
"known_question_refusal_rate": 1.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 0,
"tn": 0,
"fp": 0,
"fn": 2
}
}
}
},
"protocol_consistency_delta": {
"hop_accuracy": 0.0,
"need_precision": 0.0,
"need_recall": 0.0,
"need_f1": 0.0,
"abstention_accuracy": 0.0
}
},
"V2-512 final(step100k)": {
"checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_router_512\\memory_router_v2.pt",
"router": {
"kind": "router_v2",
"step": null,
"arch_config": {
"arch": "router_v2",
"hidden_size": 2560,
"router_dim": 512,
"num_heads": 8,
"max_hops": 3
},
"parameters": 4741902,
"checkpoint_bytes": 18973433
},
"storage": {
"address_bytes_per_record": 2048,
"address_mb_per_1m_records": 2048.0,
"checkpoint_bytes": 18973433
},
"latency": {
"cpu": {
"single_query_latency_ms_mean": 1.1631599976681173,
"single_query_latency_ms_p50": 1.1254000128246844,
"single_query_latency_ms_p95": 1.3188999728299677,
"queries_per_second": 859.7269524440167,
"samples": 5
}
},
"metrics": {
"episodes": 1940,
"answerable_episodes": 1834,
"unknown_episodes": 106,
"top1": 0.6063249727371864,
"recall1": 0.6063249727371864,
"recall3": 0.8500545256270448,
"recall5": 0.9013086150490731,
"recall8": 0.9487459105779716,
"mrr": 0.7325518172051582,
"ndcg3": 0.7456965349469862,
"mean_first_positive_rank": 2.6052344601962925,
"all_evidence_in_top3": 0.8489640130861504,
"multi_positive_episodes": 2,
"all_evidence_in_top3_multi": 0.5,
"hop_accuracy": 0.9969072164948454,
"hop_accuracy_answerable": 0.9967284623773174,
"hop_under_prediction": 0.003271537622682661,
"hop_over_prediction": 0.0,
"mean_score_margin": 3.8836888835735968,
"median_score_margin": 6.418914318084717,
"negative_margin_rate": 0.3936750272628135,
"hop_confusion": {
"true_0_pred_0": 106,
"true_0_pred_1": 0,
"true_0_pred_2": 0,
"true_1_pred_0": 4,
"true_1_pred_1": 1828,
"true_1_pred_2": 0,
"true_2_pred_0": 0,
"true_2_pred_1": 2,
"true_2_pred_2": 0
},
"known_mean_need_prob": 0.9977977412946917,
"unknown_mean_need_prob": 0.015192701093468172,
"thr0.30": {
"need_precision": 1.0,
"need_recall": 0.9978189749182116,
"need_f1": 0.9989082969432315,
"specificity_unknown_refusal": 1.0,
"abstention_accuracy": 0.9979381443298969,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.0,
"counts": {
"tp": 1830,
"tn": 106,
"fp": 0,
"fn": 4
}
},
"thr0.40": {
"need_precision": 1.0,
"need_recall": 0.9978189749182116,
"need_f1": 0.9989082969432315,
"specificity_unknown_refusal": 1.0,
"abstention_accuracy": 0.9979381443298969,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.0,
"counts": {
"tp": 1830,
"tn": 106,
"fp": 0,
"fn": 4
}
},
"thr0.50": {
"need_precision": 1.0,
"need_recall": 0.9978189749182116,
"need_f1": 0.9989082969432315,
"specificity_unknown_refusal": 1.0,
"abstention_accuracy": 0.9979381443298969,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.0,
"counts": {
"tp": 1830,
"tn": 106,
"fp": 0,
"fn": 4
}
},
"thr0.60": {
"need_precision": 1.0,
"need_recall": 0.9978189749182116,
"need_f1": 0.9989082969432315,
"specificity_unknown_refusal": 1.0,
"abstention_accuracy": 0.9979381443298969,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.0,
"counts": {
"tp": 1830,
"tn": 106,
"fp": 0,
"fn": 4
}
},
"thr0.70": {
"need_precision": 1.0,
"need_recall": 0.9978189749182116,
"need_f1": 0.9989082969432315,
"specificity_unknown_refusal": 1.0,
"abstention_accuracy": 0.9979381443298969,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.0,
"counts": {
"tp": 1830,
"tn": 106,
"fp": 0,
"fn": 4
}
}
},
"by_family": {
"benchmark_qa": {
"episodes": 64,
"answerable_episodes": 0,
"unknown_episodes": 64,
"top1": null,
"recall1": null,
"recall3": null,
"recall5": null,
"recall8": null,
"mrr": null,
"ndcg3": null,
"mean_first_positive_rank": null,
"all_evidence_in_top3": null,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": null,
"hop_under_prediction": null,
"hop_over_prediction": null,
"mean_score_margin": null,
"median_score_margin": null,
"negative_margin_rate": null,
"hop_confusion": {
"true_0_pred_0": 64
},
"known_mean_need_prob": null,
"unknown_mean_need_prob": 8.623557193993773e-08,
"thr0.50": {
"need_precision": null,
"need_recall": null,
"need_f1": null,
"specificity_unknown_refusal": 1.0,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": null,
"unknown_question_read_rate": 0.0,
"counts": {
"tp": 0,
"tn": 64,
"fp": 0,
"fn": 0
}
}
},
"mega_validation": {
"episodes": 20,
"answerable_episodes": 20,
"unknown_episodes": 0,
"top1": 0.35,
"recall1": 0.35,
"recall3": 0.8,
"recall5": 0.9,
"recall8": 0.9,
"mrr": 0.5523787878787878,
"ndcg3": 0.5633897980311373,
"mean_first_positive_rank": 4.55,
"all_evidence_in_top3": 0.7,
"multi_positive_episodes": 2,
"all_evidence_in_top3_multi": 0.5,
"hop_accuracy": 0.7,
"hop_accuracy_answerable": 0.7,
"hop_under_prediction": 0.3,
"hop_over_prediction": 0.0,
"mean_score_margin": -12.90834379196167,
"median_score_margin": -9.948484897613525,
"negative_margin_rate": 0.65,
"hop_confusion": {
"true_1_pred_0": 4,
"true_1_pred_1": 14,
"true_1_pred_2": 0,
"true_2_pred_0": 0,
"true_2_pred_1": 2,
"true_2_pred_2": 0
},
"known_mean_need_prob": 0.8000006015424048,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 0.8,
"need_f1": 0.888888888888889,
"specificity_unknown_refusal": null,
"abstention_accuracy": 0.8,
"known_question_refusal_rate": 0.2,
"unknown_question_read_rate": null,
"counts": {
"tp": 16,
"tn": 0,
"fp": 0,
"fn": 4
}
}
},
"memory_policy": {
"episodes": 1728,
"answerable_episodes": 1707,
"unknown_episodes": 21,
"top1": 0.6256590509666081,
"recall1": 0.6256590509666081,
"recall3": 0.859402460456942,
"recall5": 0.9015817223198594,
"recall8": 0.9472759226713533,
"mrr": 0.7458644125569885,
"ndcg3": 0.7599420351849566,
"mean_first_positive_rank": 2.5606326889279436,
"all_evidence_in_top3": 0.859402460456942,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 5.392943512301349,
"median_score_margin": 7.422684520483017,
"negative_margin_rate": 0.37434094903339193,
"hop_confusion": {
"true_0_pred_0": 21,
"true_0_pred_1": 0,
"true_1_pred_0": 0,
"true_1_pred_1": 1707
},
"known_mean_need_prob": 0.999977179556893,
"unknown_mean_need_prob": 0.038343352305500525,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": 1.0,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": 0.0,
"counts": {
"tp": 1707,
"tn": 21,
"fp": 0,
"fn": 0
}
}
},
"native_memory": {
"episodes": 128,
"answerable_episodes": 107,
"unknown_episodes": 21,
"top1": 0.34579439252336447,
"recall1": 0.34579439252336447,
"recall3": 0.7102803738317757,
"recall5": 0.897196261682243,
"recall8": 0.9813084112149533,
"mrr": 0.5538495772140631,
"ndcg3": 0.552510234312422,
"mean_first_positive_rank": 2.953271028037383,
"all_evidence_in_top3": 0.7102803738317757,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -17.055161562478432,
"median_score_margin": -14.022438049316406,
"negative_margin_rate": 0.6542056074766355,
"hop_confusion": {
"true_0_pred_0": 21,
"true_0_pred_1": 0,
"true_1_pred_0": 0,
"true_1_pred_1": 107
},
"known_mean_need_prob": 1.0,
"unknown_mean_need_prob": 0.038343352305500525,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": 1.0,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": 0.0,
"counts": {
"tp": 107,
"tn": 21,
"fp": 0,
"fn": 0
}
}
}
},
"by_category": {
"conflict_update": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.0,
"recall1": 0.0,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.41666666666666663,
"ndcg3": 0.5654648767857288,
"mean_first_positive_rank": 2.5,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -10.757413864135742,
"median_score_margin": -10.757413864135742,
"negative_margin_rate": 1.0,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 1.0,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"distractor_128": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.6666666666666666,
"ndcg3": 0.75,
"mean_first_positive_rank": 2.0,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -12.8532075881958,
"median_score_margin": -12.8532075881958,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.9999998807907104,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"distractor_32": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.75,
"ndcg3": 0.8154648767857288,
"mean_first_positive_rank": 1.5,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 11.88942003250122,
"median_score_margin": 11.88942003250122,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.9999871253967285,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"forget_correction": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 1.0,
"recall1": 1.0,
"recall3": 1.5,
"recall5": 2.0,
"recall8": 2.0,
"mrr": 1.0,
"ndcg3": 0.8065735963827292,
"mean_first_positive_rank": 1.0,
"all_evidence_in_top3": 0.5,
"multi_positive_episodes": 2,
"all_evidence_in_top3_multi": 0.5,
"hop_accuracy": 0.0,
"hop_accuracy_answerable": 0.0,
"hop_under_prediction": 1.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 21.42270851135254,
"median_score_margin": 21.42270851135254,
"negative_margin_rate": 0.0,
"hop_confusion": {
"true_2_pred_1": 2,
"true_2_pred_2": 0
},
"known_mean_need_prob": 0.9999992251396179,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"long_context": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.75,
"ndcg3": 0.8154648767857288,
"mean_first_positive_rank": 1.5,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -24.237293243408203,
"median_score_margin": -24.237293243408203,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 1.0,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"multi_hop": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.0,
"recall1": 0.0,
"recall3": 0.0,
"recall5": 0.0,
"recall8": 0.0,
"mrr": 0.0787878787878788,
"ndcg3": 0.0,
"mean_first_positive_rank": 13.0,
"all_evidence_in_top3": 0.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 0.0,
"hop_accuracy_answerable": 0.0,
"hop_under_prediction": 1.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -33.503143310546875,
"median_score_margin": -33.503143310546875,
"negative_margin_rate": 1.0,
"hop_confusion": {
"true_1_pred_0": 2,
"true_1_pred_1": 0
},
"known_mean_need_prob": 2.42565528196792e-06,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": null,
"need_recall": 0.0,
"need_f1": null,
"specificity_unknown_refusal": null,
"abstention_accuracy": 0.0,
"known_question_refusal_rate": 1.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 0,
"tn": 0,
"fp": 0,
"fn": 2
}
}
},
"paraphrase": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.0,
"recall1": 0.0,
"recall3": 0.0,
"recall5": 0.5,
"recall8": 0.5,
"mrr": 0.175,
"ndcg3": 0.0,
"mean_first_positive_rank": 7.0,
"all_evidence_in_top3": 0.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -41.73877143859863,
"median_score_margin": -41.73877143859863,
"negative_margin_rate": 1.0,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 1.0,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"random_position": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.75,
"ndcg3": 0.8154648767857288,
"mean_first_positive_rank": 1.5,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 31.204744338989258,
"median_score_margin": 31.204744338989258,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 1.0,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"single_fact": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.0,
"recall1": 0.0,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.41666666666666663,
"ndcg3": 0.5654648767857288,
"mean_first_positive_rank": 2.5,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -12.233797550201416,
"median_score_margin": -12.233797550201416,
"negative_margin_rate": 1.0,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 1.0,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"unknown": {
"episodes": 1920,
"answerable_episodes": 1814,
"unknown_episodes": 106,
"top1": 0.6091510474090408,
"recall1": 0.6091510474090408,
"recall3": 0.850606394707828,
"recall5": 0.9013230429988974,
"recall8": 0.9492833517089305,
"mrr": 0.7345382894138281,
"ndcg3": 0.7477065320463893,
"mean_first_positive_rank": 2.5837927232635063,
"all_evidence_in_top3": 0.850606394707828,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 4.068827060812134,
"median_score_margin": 6.489464446902275,
"negative_margin_rate": 0.3908489525909592,
"hop_confusion": {
"true_0_pred_0": 106,
"true_0_pred_1": 0,
"true_1_pred_0": 0,
"true_1_pred_1": 1814
},
"known_mean_need_prob": 0.9999785256359517,
"unknown_mean_need_prob": 0.015192701093468172,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": 1.0,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": 0.0,
"counts": {
"tp": 1814,
"tn": 106,
"fp": 0,
"fn": 0
}
}
},
"unknown_abstention": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 0.5,
"recall5": 0.5,
"recall8": 0.5,
"mrr": 0.52,
"ndcg3": 0.5,
"mean_first_positive_rank": 13.0,
"all_evidence_in_top3": 0.5,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 0.0,
"hop_accuracy_answerable": 0.0,
"hop_under_prediction": 1.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -58.27668380737305,
"median_score_margin": -58.27668380737305,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_1_pred_0": 2,
"true_1_pred_1": 0
},
"known_mean_need_prob": 1.735844170980272e-05,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": null,
"need_recall": 0.0,
"need_f1": null,
"specificity_unknown_refusal": null,
"abstention_accuracy": 0.0,
"known_question_refusal_rate": 1.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 0,
"tn": 0,
"fp": 0,
"fn": 2
}
}
}
},
"protocol_consistency_delta": {
"hop_accuracy": 0.0,
"need_precision": 0.0,
"need_recall": 0.0,
"need_f1": 0.0,
"abstention_accuracy": 0.0
}
},
"XL-512 best": {
"checkpoint": "H:\\Memory\\V2_dpskw\\checkpoints\\router_xl_512\\router_best.pt",
"router": {
"kind": "router_xl",
"step": 3500,
"arch_config": {
"arch": "router_xl",
"arch_version": 1,
"hidden_size": 2560,
"router_dim": 512,
"num_heads": 8,
"max_hops": 3,
"encoder_layers": 2,
"encoder_hidden": 512,
"pair_blocks": 1,
"pair_hidden": 512,
"pair_expansion": 2,
"pair_dropout": 0.05,
"use_interaction": true,
"policy_layers": 2,
"policy_hidden": 512,
"policy_dropout": 0.0,
"learnable_cosine_scale": true
},
"parameters": 7898127,
"checkpoint_bytes": 99014505
},
"storage": {
"address_bytes_per_record": 2048,
"address_mb_per_1m_records": 2048.0,
"checkpoint_bytes": 99014505
},
"latency": {
"cpu": {
"single_query_latency_ms_mean": 2.5489200139418244,
"single_query_latency_ms_p50": 2.477100002579391,
"single_query_latency_ms_p95": 3.3594000269658864,
"queries_per_second": 392.323020938398,
"samples": 5
}
},
"metrics": {
"episodes": 1940,
"answerable_episodes": 1834,
"unknown_episodes": 106,
"top1": 0.608505997818975,
"recall1": 0.608505997818975,
"recall3": 0.9296619411123228,
"recall5": 0.9732824427480916,
"recall8": 0.9929116684841875,
"mrr": 0.759814160403073,
"ndcg3": 0.792764395065411,
"mean_first_positive_rank": 1.83206106870229,
"all_evidence_in_top3": 0.9285714285714286,
"multi_positive_episodes": 2,
"all_evidence_in_top3_multi": 0.5,
"hop_accuracy": 0.9938144329896907,
"hop_accuracy_answerable": 0.9967284623773174,
"hop_under_prediction": 0.003271537622682661,
"hop_over_prediction": 0.0,
"mean_score_margin": 6.709678291043545,
"median_score_margin": 2.0138705372810364,
"negative_margin_rate": 0.3914940021810251,
"hop_confusion": {
"true_0_pred_0": 100,
"true_0_pred_1": 6,
"true_0_pred_2": 0,
"true_1_pred_0": 4,
"true_1_pred_1": 1828,
"true_1_pred_2": 0,
"true_2_pred_0": 0,
"true_2_pred_1": 2,
"true_2_pred_2": 0
},
"known_mean_need_prob": 0.9962027715823756,
"unknown_mean_need_prob": 0.13497692132687178,
"thr0.30": {
"need_precision": 0.9859913793103449,
"need_recall": 0.9978189749182116,
"need_f1": 0.9918699186991871,
"specificity_unknown_refusal": 0.7547169811320755,
"abstention_accuracy": 0.9845360824742269,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.24528301886792453,
"counts": {
"tp": 1830,
"tn": 80,
"fp": 26,
"fn": 4
}
},
"thr0.40": {
"need_precision": 0.9924078091106291,
"need_recall": 0.9978189749182116,
"need_f1": 0.9951060358890701,
"specificity_unknown_refusal": 0.8679245283018868,
"abstention_accuracy": 0.9907216494845361,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.1320754716981132,
"counts": {
"tp": 1830,
"tn": 92,
"fp": 14,
"fn": 4
}
},
"thr0.50": {
"need_precision": 0.9945652173913043,
"need_recall": 0.9978189749182116,
"need_f1": 0.9961894393032119,
"specificity_unknown_refusal": 0.9056603773584906,
"abstention_accuracy": 0.9927835051546392,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.09433962264150944,
"counts": {
"tp": 1830,
"tn": 96,
"fp": 10,
"fn": 4
}
},
"thr0.60": {
"need_precision": 0.9989082969432315,
"need_recall": 0.9978189749182116,
"need_f1": 0.9983633387888707,
"specificity_unknown_refusal": 0.9811320754716981,
"abstention_accuracy": 0.9969072164948454,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.018867924528301886,
"counts": {
"tp": 1830,
"tn": 104,
"fp": 2,
"fn": 4
}
},
"thr0.70": {
"need_precision": 1.0,
"need_recall": 0.9978189749182116,
"need_f1": 0.9989082969432315,
"specificity_unknown_refusal": 1.0,
"abstention_accuracy": 0.9979381443298969,
"known_question_refusal_rate": 0.0021810250817884407,
"unknown_question_read_rate": 0.0,
"counts": {
"tp": 1830,
"tn": 106,
"fp": 0,
"fn": 4
}
}
},
"by_family": {
"benchmark_qa": {
"episodes": 64,
"answerable_episodes": 0,
"unknown_episodes": 64,
"top1": null,
"recall1": null,
"recall3": null,
"recall5": null,
"recall8": null,
"mrr": null,
"ndcg3": null,
"mean_first_positive_rank": null,
"all_evidence_in_top3": null,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": null,
"hop_under_prediction": null,
"hop_over_prediction": null,
"mean_score_margin": null,
"median_score_margin": null,
"negative_margin_rate": null,
"hop_confusion": {
"true_0_pred_0": 64
},
"known_mean_need_prob": null,
"unknown_mean_need_prob": 0.003423646275223291,
"thr0.50": {
"need_precision": null,
"need_recall": null,
"need_f1": null,
"specificity_unknown_refusal": 1.0,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": null,
"unknown_question_read_rate": 0.0,
"counts": {
"tp": 0,
"tn": 64,
"fp": 0,
"fn": 0
}
}
},
"mega_validation": {
"episodes": 20,
"answerable_episodes": 20,
"unknown_episodes": 0,
"top1": 0.4,
"recall1": 0.4,
"recall3": 0.85,
"recall5": 0.9,
"recall8": 0.9,
"mrr": 0.6083352272727273,
"ndcg3": 0.6361540935691467,
"mean_first_positive_rank": 6.15,
"all_evidence_in_top3": 0.75,
"multi_positive_episodes": 2,
"all_evidence_in_top3_multi": 0.5,
"hop_accuracy": 0.7,
"hop_accuracy_answerable": 0.7,
"hop_under_prediction": 0.3,
"hop_over_prediction": 0.0,
"mean_score_margin": -0.6135697185993194,
"median_score_margin": -0.7090378999710083,
"negative_margin_rate": 0.6,
"hop_confusion": {
"true_1_pred_0": 4,
"true_1_pred_1": 14,
"true_1_pred_2": 0,
"true_2_pred_0": 0,
"true_2_pred_1": 2,
"true_2_pred_2": 0
},
"known_mean_need_prob": 0.8056919909082353,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 0.8,
"need_f1": 0.888888888888889,
"specificity_unknown_refusal": null,
"abstention_accuracy": 0.8,
"known_question_refusal_rate": 0.2,
"unknown_question_read_rate": null,
"counts": {
"tp": 16,
"tn": 0,
"fp": 0,
"fn": 4
}
}
},
"memory_policy": {
"episodes": 1728,
"answerable_episodes": 1707,
"unknown_episodes": 21,
"top1": 0.6239015817223199,
"recall1": 0.6239015817223199,
"recall3": 0.9420035149384886,
"recall5": 0.9806678383128296,
"recall8": 0.9953134153485648,
"mrr": 0.7716295930129063,
"ndcg3": 0.8064999615386277,
"mean_first_positive_rank": 1.7258347978910369,
"all_evidence_in_top3": 0.9420035149384886,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 0.9982638888888888,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 7.258645896374848,
"median_score_margin": 2.5352253913879395,
"negative_margin_rate": 0.37609841827768015,
"hop_confusion": {
"true_0_pred_0": 18,
"true_0_pred_1": 3,
"true_1_pred_0": 0,
"true_1_pred_1": 1707
},
"known_mean_need_prob": 0.9981995297581673,
"unknown_mean_need_prob": 0.33543905473890756,
"thr0.50": {
"need_precision": 0.9970794392523364,
"need_recall": 1.0,
"need_f1": 0.9985375840889148,
"specificity_unknown_refusal": 0.7619047619047619,
"abstention_accuracy": 0.9971064814814815,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": 0.23809523809523808,
"counts": {
"tp": 1707,
"tn": 16,
"fp": 5,
"fn": 0
}
}
},
"native_memory": {
"episodes": 128,
"answerable_episodes": 107,
"unknown_episodes": 21,
"top1": 0.40186915887850466,
"recall1": 0.40186915887850466,
"recall3": 0.7476635514018691,
"recall5": 0.8691588785046729,
"recall8": 0.9719626168224299,
"mrr": 0.5996331809415921,
"ndcg3": 0.6029101339452666,
"mean_first_positive_rank": 2.7196261682242993,
"all_evidence_in_top3": 0.7476635514018691,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 0.9765625,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -0.6793192987478225,
"median_score_margin": -0.7896833419799805,
"negative_margin_rate": 0.5981308411214953,
"hop_confusion": {
"true_0_pred_0": 18,
"true_0_pred_1": 3,
"true_1_pred_0": 0,
"true_1_pred_1": 107
},
"known_mean_need_prob": 0.9999574389413138,
"unknown_mean_need_prob": 0.33543905473890756,
"thr0.50": {
"need_precision": 0.9553571428571429,
"need_recall": 1.0,
"need_f1": 0.9771689497716896,
"specificity_unknown_refusal": 0.7619047619047619,
"abstention_accuracy": 0.9609375,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": 0.23809523809523808,
"counts": {
"tp": 107,
"tn": 16,
"fp": 5,
"fn": 0
}
}
}
},
"by_category": {
"conflict_update": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.75,
"ndcg3": 0.8154648767857288,
"mean_first_positive_rank": 1.5,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 1.907712697982788,
"median_score_margin": 1.907712697982788,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.9999270439147949,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"distractor_128": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 1.0,
"recall1": 1.0,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 1.0,
"ndcg3": 1.0,
"mean_first_positive_rank": 1.0,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 4.908966302871704,
"median_score_margin": 4.908966302871704,
"negative_margin_rate": 0.0,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.9988398551940918,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"distractor_32": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.75,
"ndcg3": 0.8154648767857288,
"mean_first_positive_rank": 1.5,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 7.163174152374268,
"median_score_margin": 7.163174152374268,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.9931761622428894,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"forget_correction": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 1.5,
"recall5": 2.0,
"recall8": 2.0,
"mrr": 0.75,
"ndcg3": 0.6532867981913646,
"mean_first_positive_rank": 1.5,
"all_evidence_in_top3": 0.5,
"multi_positive_episodes": 2,
"all_evidence_in_top3_multi": 0.5,
"hop_accuracy": 0.0,
"hop_accuracy_answerable": 0.0,
"hop_under_prediction": 1.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 4.6119712591171265,
"median_score_margin": 4.6119712591171265,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_2_pred_1": 2,
"true_2_pred_2": 0
},
"known_mean_need_prob": 0.998505711555481,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"long_context": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.75,
"ndcg3": 0.8154648767857288,
"mean_first_positive_rank": 1.5,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -1.0701351165771484,
"median_score_margin": -1.0701351165771484,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.9997996389865875,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"multi_hop": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.0,
"recall1": 0.0,
"recall3": 0.0,
"recall5": 0.0,
"recall8": 0.0,
"mrr": 0.045,
"ndcg3": 0.0,
"mean_first_positive_rank": 22.5,
"all_evidence_in_top3": 0.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 0.0,
"hop_accuracy_answerable": 0.0,
"hop_under_prediction": 1.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -13.473207950592041,
"median_score_margin": -13.473207950592041,
"negative_margin_rate": 1.0,
"hop_confusion": {
"true_1_pred_0": 2,
"true_1_pred_1": 0
},
"known_mean_need_prob": 0.027262956835329533,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": null,
"need_recall": 0.0,
"need_f1": null,
"specificity_unknown_refusal": null,
"abstention_accuracy": 0.0,
"known_question_refusal_rate": 1.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 0,
"tn": 0,
"fp": 0,
"fn": 2
}
}
},
"paraphrase": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.75,
"ndcg3": 0.8154648767857288,
"mean_first_positive_rank": 1.5,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 5.6224164962768555,
"median_score_margin": 5.6224164962768555,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.9998534619808197,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"random_position": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.5,
"recall1": 0.5,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.75,
"ndcg3": 0.8154648767857288,
"mean_first_positive_rank": 1.5,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 5.72838020324707,
"median_score_margin": 5.72838020324707,
"negative_margin_rate": 0.5,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.9995926022529602,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"single_fact": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.0,
"recall1": 0.0,
"recall3": 1.0,
"recall5": 1.0,
"recall8": 1.0,
"mrr": 0.5,
"ndcg3": 0.6309297535714575,
"mean_first_positive_rank": 2.0,
"all_evidence_in_top3": 1.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 1.0,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -0.9848489761352539,
"median_score_margin": -0.9848489761352539,
"negative_margin_rate": 1.0,
"hop_confusion": {
"true_1_pred_1": 2
},
"known_mean_need_prob": 0.9999998807907104,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": 1.0,
"need_recall": 1.0,
"need_f1": 1.0,
"specificity_unknown_refusal": null,
"abstention_accuracy": 1.0,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 2,
"tn": 0,
"fp": 0,
"fn": 0
}
}
},
"unknown": {
"episodes": 1920,
"answerable_episodes": 1814,
"unknown_episodes": 106,
"top1": 0.6108048511576626,
"recall1": 0.6108048511576626,
"recall3": 0.9305402425578831,
"recall5": 0.974090407938258,
"recall8": 0.9939360529217199,
"mrr": 0.7614842699193943,
"ndcg3": 0.7944910797566598,
"mean_first_positive_rank": 1.7844542447629548,
"all_evidence_in_top3": 0.9305402425578831,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 0.996875,
"hop_accuracy_answerable": 1.0,
"hop_under_prediction": 0.0,
"hop_over_prediction": 0.0,
"mean_score_margin": 6.790419724446443,
"median_score_margin": 2.042374363169074,
"negative_margin_rate": 0.38919514884233736,
"hop_confusion": {
"true_0_pred_0": 100,
"true_0_pred_1": 6,
"true_1_pred_0": 0,
"true_1_pred_1": 1814
},
"known_mean_need_prob": 0.9983032212039207,
"unknown_mean_need_prob": 0.13497692132687178,
"thr0.50": {
"need_precision": 0.9945175438596491,
"need_recall": 1.0,
"need_f1": 0.9972512369433755,
"specificity_unknown_refusal": 0.9056603773584906,
"abstention_accuracy": 0.9947916666666666,
"known_question_refusal_rate": 0.0,
"unknown_question_read_rate": 0.09433962264150944,
"counts": {
"tp": 1814,
"tn": 96,
"fp": 10,
"fn": 0
}
}
},
"unknown_abstention": {
"episodes": 2,
"answerable_episodes": 2,
"unknown_episodes": 0,
"top1": 0.0,
"recall1": 0.0,
"recall3": 0.0,
"recall5": 0.0,
"recall8": 0.0,
"mrr": 0.03835227272727273,
"ndcg3": 0.0,
"mean_first_positive_rank": 27.0,
"all_evidence_in_top3": 0.0,
"multi_positive_episodes": 0,
"all_evidence_in_top3_multi": null,
"hop_accuracy": 0.0,
"hop_accuracy_answerable": 0.0,
"hop_under_prediction": 1.0,
"hop_over_prediction": 0.0,
"mean_score_margin": -20.550126254558563,
"median_score_margin": -20.550126254558563,
"negative_margin_rate": 1.0,
"hop_confusion": {
"true_1_pred_0": 2,
"true_1_pred_1": 0
},
"known_mean_need_prob": 0.03996259532868862,
"unknown_mean_need_prob": null,
"thr0.50": {
"need_precision": null,
"need_recall": 0.0,
"need_f1": null,
"specificity_unknown_refusal": null,
"abstention_accuracy": 0.0,
"known_question_refusal_rate": 1.0,
"unknown_question_read_rate": null,
"counts": {
"tp": 0,
"tn": 0,
"fp": 0,
"fn": 2
}
}
}
},
"protocol_consistency_delta": {
"hop_accuracy": 0.0,
"need_precision": 0.0,
"need_recall": 0.0,
"need_f1": 0.0,
"abstention_accuracy": 0.0
}
}
}
Reference in New Issue View Git Blame Copy Permalink