WpyQwq
643e22ecb9
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线 - 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同） - 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00） - 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00% - 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益） - 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
2456 lines
78 KiB
JSON
2456 lines
78 KiB
JSON
{
|
|
"V2-512 best(step1000)": {
|
|
"checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_router_512\\router_best.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": 1000,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 4741902,
|
|
"checkpoint_bytes": 56922381
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048.0,
|
|
"checkpoint_bytes": 56922381
|
|
},
|
|
"latency": {
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 1.2766800005920231,
|
|
"single_query_latency_ms_p50": 1.2820999836549163,
|
|
"single_query_latency_ms_p95": 1.3297999976202846,
|
|
"queries_per_second": 783.2816363820841,
|
|
"samples": 5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 1940,
|
|
"answerable_episodes": 1834,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.6712104689203926,
|
|
"recall1": 0.6712104689203926,
|
|
"recall3": 0.8876772082878953,
|
|
"recall5": 0.9492911668484187,
|
|
"recall8": 0.9858233369683751,
|
|
"mrr": 0.7908385671852008,
|
|
"ndcg3": 0.7999554748364778,
|
|
"mean_first_positive_rank": 1.841875681570338,
|
|
"all_evidence_in_top3": 0.8865866957470011,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.9783505154639175,
|
|
"hop_accuracy_answerable": 0.9967284623773174,
|
|
"hop_under_prediction": 0.003271537622682661,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.5880087030850424,
|
|
"median_score_margin": 0.347680926322937,
|
|
"negative_margin_rate": 0.3287895310796074,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 70,
|
|
"true_0_pred_1": 36,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 4,
|
|
"true_1_pred_1": 1828,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.9847397789147844,
|
|
"unknown_mean_need_prob": 0.2536155520674755,
|
|
"thr0.30": {
|
|
"need_precision": 0.9786096256684492,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9881209503239742,
|
|
"specificity_unknown_refusal": 0.6226415094339622,
|
|
"abstention_accuracy": 0.977319587628866,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.37735849056603776,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 66,
|
|
"fp": 40,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.9796573875802997,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9886547811993517,
|
|
"specificity_unknown_refusal": 0.6415094339622641,
|
|
"abstention_accuracy": 0.9783505154639175,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.3584905660377358,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 68,
|
|
"fp": 38,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.9807073954983923,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9891891891891892,
|
|
"specificity_unknown_refusal": 0.660377358490566,
|
|
"abstention_accuracy": 0.979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.33962264150943394,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 70,
|
|
"fp": 36,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.9902597402597403,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9940249864204237,
|
|
"specificity_unknown_refusal": 0.8301886792452831,
|
|
"abstention_accuracy": 0.988659793814433,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.16981132075471697,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 88,
|
|
"fp": 18,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 0.9989082969432315,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9983633387888707,
|
|
"specificity_unknown_refusal": 0.9811320754716981,
|
|
"abstention_accuracy": 0.9969072164948454,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.018867924528301886,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 104,
|
|
"fp": 2,
|
|
"fn": 4
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.04612289386568591,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20,
|
|
"answerable_episodes": 20,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.4,
|
|
"recall1": 0.4,
|
|
"recall3": 0.8,
|
|
"recall5": 1.0,
|
|
"recall8": 1.05,
|
|
"mrr": 0.6225574712643678,
|
|
"ndcg3": 0.6086215664331643,
|
|
"mean_first_positive_rank": 3.45,
|
|
"all_evidence_in_top3": 0.7,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.7,
|
|
"hop_accuracy_answerable": 0.7,
|
|
"hop_under_prediction": 0.3,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.29634838700294497,
|
|
"median_score_margin": -0.25011515617370605,
|
|
"negative_margin_rate": 0.6,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 4,
|
|
"true_1_pred_1": 14,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.8251703515648842,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8,
|
|
"need_f1": 0.888888888888889,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 0.8,
|
|
"known_question_refusal_rate": 0.2,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 16,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.6988869361452841,
|
|
"recall1": 0.6988869361452841,
|
|
"recall3": 0.9004100761570006,
|
|
"recall5": 0.9537199765670767,
|
|
"recall8": 0.9859402460456942,
|
|
"mrr": 0.8099543326670381,
|
|
"ndcg3": 0.8196676424617168,
|
|
"mean_first_positive_rank": 1.7492677211482133,
|
|
"all_evidence_in_top3": 0.9004100761570006,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.9895833333333334,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.6516404748137541,
|
|
"median_score_margin": 0.38489866256713867,
|
|
"negative_margin_rate": 0.30111306385471587,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 3,
|
|
"true_0_pred_1": 18,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9862654947298761,
|
|
"unknown_mean_need_prob": 0.5697948407559168,
|
|
"thr0.50": {
|
|
"need_precision": 0.9895652173913043,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9947552447552448,
|
|
"specificity_unknown_refusal": 0.14285714285714285,
|
|
"abstention_accuracy": 0.9895833333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.8571428571428571,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 3,
|
|
"fp": 18,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.2803738317757009,
|
|
"recall1": 0.2803738317757009,
|
|
"recall3": 0.7009345794392523,
|
|
"recall5": 0.8691588785046729,
|
|
"recall8": 0.9719626168224299,
|
|
"mrr": 0.5173339899975414,
|
|
"ndcg3": 0.5212452695260414,
|
|
"mean_first_positive_rank": 3.0186915887850465,
|
|
"all_evidence_in_top3": 0.7009345794392523,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.859375,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.26182580662664967,
|
|
"median_score_margin": -0.27979564666748047,
|
|
"negative_margin_rate": 0.719626168224299,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 3,
|
|
"true_0_pred_1": 18,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 0.9902256821917597,
|
|
"unknown_mean_need_prob": 0.5697948407559168,
|
|
"thr0.50": {
|
|
"need_precision": 0.856,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9224137931034483,
|
|
"specificity_unknown_refusal": 0.14285714285714285,
|
|
"abstention_accuracy": 0.859375,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.8571428571428571,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 3,
|
|
"fp": 18,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.25075769424438477,
|
|
"median_score_margin": 0.25075769424438477,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9954056441783905,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 1.0,
|
|
"recall1": 1.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 1.0,
|
|
"ndcg3": 1.0,
|
|
"mean_first_positive_rank": 1.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 3.3609426021575928,
|
|
"median_score_margin": 3.3609426021575928,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.971575915813446,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 1.208745002746582,
|
|
"median_score_margin": 1.208745002746582,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.92432501912117,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.5,
|
|
"recall5": 2.0,
|
|
"recall8": 2.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.6934264036172708,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 10.865452527999878,
|
|
"median_score_margin": 10.865452527999878,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.9887627363204956,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 0.5,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.625,
|
|
"ndcg3": 0.5,
|
|
"mean_first_positive_rank": 2.5,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -1.6004953384399414,
|
|
"median_score_margin": -1.6004953384399414,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9807772934436798,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.5,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.375,
|
|
"ndcg3": 0.31546487678572877,
|
|
"mean_first_positive_rank": 3.0,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -3.4204822182655334,
|
|
"median_score_margin": -3.4204822182655334,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 2,
|
|
"true_1_pred_1": 0
|
|
},
|
|
"known_mean_need_prob": 0.1579926311969757,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": 0.0,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.5,
|
|
"recall5": 0.5,
|
|
"recall8": 1.0,
|
|
"mrr": 0.3333333333333333,
|
|
"ndcg3": 0.31546487678572877,
|
|
"mean_first_positive_rank": 4.0,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -2.652513027191162,
|
|
"median_score_margin": -2.652513027191162,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.99253049492836,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 1.0,
|
|
"recall1": 1.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 1.0,
|
|
"ndcg3": 1.0,
|
|
"mean_first_positive_rank": 1.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 3.1475493907928467,
|
|
"median_score_margin": 3.1475493907928467,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9879558682441711,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.5,
|
|
"ndcg3": 0.6309297535714575,
|
|
"mean_first_positive_rank": 2.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.12020683288574219,
|
|
"median_score_margin": -0.12020683288574219,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9997169077396393,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.6742006615214995,
|
|
"recall1": 0.6742006615214995,
|
|
"recall3": 0.8886438809261301,
|
|
"recall5": 0.9487320837927232,
|
|
"recall8": 0.9851157662624035,
|
|
"mrr": 0.792693926566908,
|
|
"ndcg3": 0.8020649997361834,
|
|
"mean_first_positive_rank": 1.8241455347298787,
|
|
"all_evidence_in_top3": 0.8886438809261301,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.98125,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.5977590568897612,
|
|
"median_score_margin": 0.3488118648529053,
|
|
"negative_margin_rate": 0.32579933847850057,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 70,
|
|
"true_0_pred_1": 36,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9864990890288957,
|
|
"unknown_mean_need_prob": 0.2536155520674755,
|
|
"thr0.50": {
|
|
"need_precision": 0.9805405405405405,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.990174672489083,
|
|
"specificity_unknown_refusal": 0.660377358490566,
|
|
"abstention_accuracy": 0.98125,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.33962264150943394,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 70,
|
|
"fp": 36,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.5,
|
|
"recall8": 0.5,
|
|
"mrr": 0.14224137931034483,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 16.5,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -14.003233671188354,
|
|
"median_score_margin": -14.003233671188354,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 2,
|
|
"true_1_pred_1": 0
|
|
},
|
|
"known_mean_need_prob": 0.25266100466251373,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": 0.0,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"protocol_consistency_delta": {
|
|
"hop_accuracy": 0.0,
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"abstention_accuracy": 0.0
|
|
}
|
|
},
|
|
"V2-512 final(step100k)": {
|
|
"checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_router_512\\memory_router_v2.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 4741902,
|
|
"checkpoint_bytes": 18973433
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048.0,
|
|
"checkpoint_bytes": 18973433
|
|
},
|
|
"latency": {
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 1.1631599976681173,
|
|
"single_query_latency_ms_p50": 1.1254000128246844,
|
|
"single_query_latency_ms_p95": 1.3188999728299677,
|
|
"queries_per_second": 859.7269524440167,
|
|
"samples": 5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 1940,
|
|
"answerable_episodes": 1834,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.6063249727371864,
|
|
"recall1": 0.6063249727371864,
|
|
"recall3": 0.8500545256270448,
|
|
"recall5": 0.9013086150490731,
|
|
"recall8": 0.9487459105779716,
|
|
"mrr": 0.7325518172051582,
|
|
"ndcg3": 0.7456965349469862,
|
|
"mean_first_positive_rank": 2.6052344601962925,
|
|
"all_evidence_in_top3": 0.8489640130861504,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.9969072164948454,
|
|
"hop_accuracy_answerable": 0.9967284623773174,
|
|
"hop_under_prediction": 0.003271537622682661,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 3.8836888835735968,
|
|
"median_score_margin": 6.418914318084717,
|
|
"negative_margin_rate": 0.3936750272628135,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 4,
|
|
"true_1_pred_1": 1828,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.9977977412946917,
|
|
"unknown_mean_need_prob": 0.015192701093468172,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 8.623557193993773e-08,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20,
|
|
"answerable_episodes": 20,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.35,
|
|
"recall1": 0.35,
|
|
"recall3": 0.8,
|
|
"recall5": 0.9,
|
|
"recall8": 0.9,
|
|
"mrr": 0.5523787878787878,
|
|
"ndcg3": 0.5633897980311373,
|
|
"mean_first_positive_rank": 4.55,
|
|
"all_evidence_in_top3": 0.7,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.7,
|
|
"hop_accuracy_answerable": 0.7,
|
|
"hop_under_prediction": 0.3,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -12.90834379196167,
|
|
"median_score_margin": -9.948484897613525,
|
|
"negative_margin_rate": 0.65,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 4,
|
|
"true_1_pred_1": 14,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.8000006015424048,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8,
|
|
"need_f1": 0.888888888888889,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 0.8,
|
|
"known_question_refusal_rate": 0.2,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 16,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.6256590509666081,
|
|
"recall1": 0.6256590509666081,
|
|
"recall3": 0.859402460456942,
|
|
"recall5": 0.9015817223198594,
|
|
"recall8": 0.9472759226713533,
|
|
"mrr": 0.7458644125569885,
|
|
"ndcg3": 0.7599420351849566,
|
|
"mean_first_positive_rank": 2.5606326889279436,
|
|
"all_evidence_in_top3": 0.859402460456942,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 5.392943512301349,
|
|
"median_score_margin": 7.422684520483017,
|
|
"negative_margin_rate": 0.37434094903339193,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.999977179556893,
|
|
"unknown_mean_need_prob": 0.038343352305500525,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.34579439252336447,
|
|
"recall1": 0.34579439252336447,
|
|
"recall3": 0.7102803738317757,
|
|
"recall5": 0.897196261682243,
|
|
"recall8": 0.9813084112149533,
|
|
"mrr": 0.5538495772140631,
|
|
"ndcg3": 0.552510234312422,
|
|
"mean_first_positive_rank": 2.953271028037383,
|
|
"all_evidence_in_top3": 0.7102803738317757,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -17.055161562478432,
|
|
"median_score_margin": -14.022438049316406,
|
|
"negative_margin_rate": 0.6542056074766355,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.038343352305500525,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.41666666666666663,
|
|
"ndcg3": 0.5654648767857288,
|
|
"mean_first_positive_rank": 2.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -10.757413864135742,
|
|
"median_score_margin": -10.757413864135742,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.6666666666666666,
|
|
"ndcg3": 0.75,
|
|
"mean_first_positive_rank": 2.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -12.8532075881958,
|
|
"median_score_margin": -12.8532075881958,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9999998807907104,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 11.88942003250122,
|
|
"median_score_margin": 11.88942003250122,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9999871253967285,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 1.0,
|
|
"recall1": 1.0,
|
|
"recall3": 1.5,
|
|
"recall5": 2.0,
|
|
"recall8": 2.0,
|
|
"mrr": 1.0,
|
|
"ndcg3": 0.8065735963827292,
|
|
"mean_first_positive_rank": 1.0,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 21.42270851135254,
|
|
"median_score_margin": 21.42270851135254,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.9999992251396179,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -24.237293243408203,
|
|
"median_score_margin": -24.237293243408203,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.0,
|
|
"recall8": 0.0,
|
|
"mrr": 0.0787878787878788,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 13.0,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -33.503143310546875,
|
|
"median_score_margin": -33.503143310546875,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 2,
|
|
"true_1_pred_1": 0
|
|
},
|
|
"known_mean_need_prob": 2.42565528196792e-06,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": 0.0,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.5,
|
|
"recall8": 0.5,
|
|
"mrr": 0.175,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 7.0,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -41.73877143859863,
|
|
"median_score_margin": -41.73877143859863,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 31.204744338989258,
|
|
"median_score_margin": 31.204744338989258,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.41666666666666663,
|
|
"ndcg3": 0.5654648767857288,
|
|
"mean_first_positive_rank": 2.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -12.233797550201416,
|
|
"median_score_margin": -12.233797550201416,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.6091510474090408,
|
|
"recall1": 0.6091510474090408,
|
|
"recall3": 0.850606394707828,
|
|
"recall5": 0.9013230429988974,
|
|
"recall8": 0.9492833517089305,
|
|
"mrr": 0.7345382894138281,
|
|
"ndcg3": 0.7477065320463893,
|
|
"mean_first_positive_rank": 2.5837927232635063,
|
|
"all_evidence_in_top3": 0.850606394707828,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 4.068827060812134,
|
|
"median_score_margin": 6.489464446902275,
|
|
"negative_margin_rate": 0.3908489525909592,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9999785256359517,
|
|
"unknown_mean_need_prob": 0.015192701093468172,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 0.5,
|
|
"recall5": 0.5,
|
|
"recall8": 0.5,
|
|
"mrr": 0.52,
|
|
"ndcg3": 0.5,
|
|
"mean_first_positive_rank": 13.0,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -58.27668380737305,
|
|
"median_score_margin": -58.27668380737305,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 2,
|
|
"true_1_pred_1": 0
|
|
},
|
|
"known_mean_need_prob": 1.735844170980272e-05,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": 0.0,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"protocol_consistency_delta": {
|
|
"hop_accuracy": 0.0,
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"abstention_accuracy": 0.0
|
|
}
|
|
},
|
|
"XL-512 best": {
|
|
"checkpoint": "H:\\Memory\\V2_dpskw\\checkpoints\\router_xl_512\\router_best.pt",
|
|
"router": {
|
|
"kind": "router_xl",
|
|
"step": 3500,
|
|
"arch_config": {
|
|
"arch": "router_xl",
|
|
"arch_version": 1,
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3,
|
|
"encoder_layers": 2,
|
|
"encoder_hidden": 512,
|
|
"pair_blocks": 1,
|
|
"pair_hidden": 512,
|
|
"pair_expansion": 2,
|
|
"pair_dropout": 0.05,
|
|
"use_interaction": true,
|
|
"policy_layers": 2,
|
|
"policy_hidden": 512,
|
|
"policy_dropout": 0.0,
|
|
"learnable_cosine_scale": true
|
|
},
|
|
"parameters": 7898127,
|
|
"checkpoint_bytes": 99014505
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048.0,
|
|
"checkpoint_bytes": 99014505
|
|
},
|
|
"latency": {
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 2.5489200139418244,
|
|
"single_query_latency_ms_p50": 2.477100002579391,
|
|
"single_query_latency_ms_p95": 3.3594000269658864,
|
|
"queries_per_second": 392.323020938398,
|
|
"samples": 5
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 1940,
|
|
"answerable_episodes": 1834,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.608505997818975,
|
|
"recall1": 0.608505997818975,
|
|
"recall3": 0.9296619411123228,
|
|
"recall5": 0.9732824427480916,
|
|
"recall8": 0.9929116684841875,
|
|
"mrr": 0.759814160403073,
|
|
"ndcg3": 0.792764395065411,
|
|
"mean_first_positive_rank": 1.83206106870229,
|
|
"all_evidence_in_top3": 0.9285714285714286,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.9938144329896907,
|
|
"hop_accuracy_answerable": 0.9967284623773174,
|
|
"hop_under_prediction": 0.003271537622682661,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 6.709678291043545,
|
|
"median_score_margin": 2.0138705372810364,
|
|
"negative_margin_rate": 0.3914940021810251,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 100,
|
|
"true_0_pred_1": 6,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 4,
|
|
"true_1_pred_1": 1828,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.9962027715823756,
|
|
"unknown_mean_need_prob": 0.13497692132687178,
|
|
"thr0.30": {
|
|
"need_precision": 0.9859913793103449,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9918699186991871,
|
|
"specificity_unknown_refusal": 0.7547169811320755,
|
|
"abstention_accuracy": 0.9845360824742269,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.24528301886792453,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 80,
|
|
"fp": 26,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.9924078091106291,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9951060358890701,
|
|
"specificity_unknown_refusal": 0.8679245283018868,
|
|
"abstention_accuracy": 0.9907216494845361,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.1320754716981132,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 92,
|
|
"fp": 14,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.9945652173913043,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9961894393032119,
|
|
"specificity_unknown_refusal": 0.9056603773584906,
|
|
"abstention_accuracy": 0.9927835051546392,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.09433962264150944,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 96,
|
|
"fp": 10,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.9989082969432315,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9983633387888707,
|
|
"specificity_unknown_refusal": 0.9811320754716981,
|
|
"abstention_accuracy": 0.9969072164948454,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.018867924528301886,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 104,
|
|
"fp": 2,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.003423646275223291,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20,
|
|
"answerable_episodes": 20,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.4,
|
|
"recall1": 0.4,
|
|
"recall3": 0.85,
|
|
"recall5": 0.9,
|
|
"recall8": 0.9,
|
|
"mrr": 0.6083352272727273,
|
|
"ndcg3": 0.6361540935691467,
|
|
"mean_first_positive_rank": 6.15,
|
|
"all_evidence_in_top3": 0.75,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.7,
|
|
"hop_accuracy_answerable": 0.7,
|
|
"hop_under_prediction": 0.3,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.6135697185993194,
|
|
"median_score_margin": -0.7090378999710083,
|
|
"negative_margin_rate": 0.6,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 4,
|
|
"true_1_pred_1": 14,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.8056919909082353,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8,
|
|
"need_f1": 0.888888888888889,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 0.8,
|
|
"known_question_refusal_rate": 0.2,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 16,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.6239015817223199,
|
|
"recall1": 0.6239015817223199,
|
|
"recall3": 0.9420035149384886,
|
|
"recall5": 0.9806678383128296,
|
|
"recall8": 0.9953134153485648,
|
|
"mrr": 0.7716295930129063,
|
|
"ndcg3": 0.8064999615386277,
|
|
"mean_first_positive_rank": 1.7258347978910369,
|
|
"all_evidence_in_top3": 0.9420035149384886,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.9982638888888888,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 7.258645896374848,
|
|
"median_score_margin": 2.5352253913879395,
|
|
"negative_margin_rate": 0.37609841827768015,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 18,
|
|
"true_0_pred_1": 3,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9981995297581673,
|
|
"unknown_mean_need_prob": 0.33543905473890756,
|
|
"thr0.50": {
|
|
"need_precision": 0.9970794392523364,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9985375840889148,
|
|
"specificity_unknown_refusal": 0.7619047619047619,
|
|
"abstention_accuracy": 0.9971064814814815,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.23809523809523808,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 16,
|
|
"fp": 5,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.40186915887850466,
|
|
"recall1": 0.40186915887850466,
|
|
"recall3": 0.7476635514018691,
|
|
"recall5": 0.8691588785046729,
|
|
"recall8": 0.9719626168224299,
|
|
"mrr": 0.5996331809415921,
|
|
"ndcg3": 0.6029101339452666,
|
|
"mean_first_positive_rank": 2.7196261682242993,
|
|
"all_evidence_in_top3": 0.7476635514018691,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.9765625,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.6793192987478225,
|
|
"median_score_margin": -0.7896833419799805,
|
|
"negative_margin_rate": 0.5981308411214953,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 18,
|
|
"true_0_pred_1": 3,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 0.9999574389413138,
|
|
"unknown_mean_need_prob": 0.33543905473890756,
|
|
"thr0.50": {
|
|
"need_precision": 0.9553571428571429,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9771689497716896,
|
|
"specificity_unknown_refusal": 0.7619047619047619,
|
|
"abstention_accuracy": 0.9609375,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.23809523809523808,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 16,
|
|
"fp": 5,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 1.907712697982788,
|
|
"median_score_margin": 1.907712697982788,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9999270439147949,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 1.0,
|
|
"recall1": 1.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 1.0,
|
|
"ndcg3": 1.0,
|
|
"mean_first_positive_rank": 1.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 4.908966302871704,
|
|
"median_score_margin": 4.908966302871704,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9988398551940918,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 7.163174152374268,
|
|
"median_score_margin": 7.163174152374268,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9931761622428894,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.5,
|
|
"recall5": 2.0,
|
|
"recall8": 2.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.6532867981913646,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 4.6119712591171265,
|
|
"median_score_margin": 4.6119712591171265,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.998505711555481,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -1.0701351165771484,
|
|
"median_score_margin": -1.0701351165771484,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9997996389865875,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.0,
|
|
"recall8": 0.0,
|
|
"mrr": 0.045,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 22.5,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -13.473207950592041,
|
|
"median_score_margin": -13.473207950592041,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 2,
|
|
"true_1_pred_1": 0
|
|
},
|
|
"known_mean_need_prob": 0.027262956835329533,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": 0.0,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 5.6224164962768555,
|
|
"median_score_margin": 5.6224164962768555,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9998534619808197,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 5.72838020324707,
|
|
"median_score_margin": 5.72838020324707,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9995926022529602,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.5,
|
|
"ndcg3": 0.6309297535714575,
|
|
"mean_first_positive_rank": 2.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.9848489761352539,
|
|
"median_score_margin": -0.9848489761352539,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9999998807907104,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.6108048511576626,
|
|
"recall1": 0.6108048511576626,
|
|
"recall3": 0.9305402425578831,
|
|
"recall5": 0.974090407938258,
|
|
"recall8": 0.9939360529217199,
|
|
"mrr": 0.7614842699193943,
|
|
"ndcg3": 0.7944910797566598,
|
|
"mean_first_positive_rank": 1.7844542447629548,
|
|
"all_evidence_in_top3": 0.9305402425578831,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.996875,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 6.790419724446443,
|
|
"median_score_margin": 2.042374363169074,
|
|
"negative_margin_rate": 0.38919514884233736,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 100,
|
|
"true_0_pred_1": 6,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9983032212039207,
|
|
"unknown_mean_need_prob": 0.13497692132687178,
|
|
"thr0.50": {
|
|
"need_precision": 0.9945175438596491,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9972512369433755,
|
|
"specificity_unknown_refusal": 0.9056603773584906,
|
|
"abstention_accuracy": 0.9947916666666666,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.09433962264150944,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 96,
|
|
"fp": 10,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.0,
|
|
"recall8": 0.0,
|
|
"mrr": 0.03835227272727273,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 27.0,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -20.550126254558563,
|
|
"median_score_margin": -20.550126254558563,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 2,
|
|
"true_1_pred_1": 0
|
|
},
|
|
"known_mean_need_prob": 0.03996259532868862,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": 0.0,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"protocol_consistency_delta": {
|
|
"hop_accuracy": 0.0,
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"abstention_accuracy": 0.0
|
|
}
|
|
}
|
|
} |