WpyQwq
643e22ecb9
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线 - 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同） - 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00） - 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00% - 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益） - 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
1743 lines
56 KiB
JSON
1743 lines
56 KiB
JSON
{
|
|
"V2-128 v6 final": {
|
|
"checkpoint": "checkpoints\\router_v6_v2_128\\memory_router_v2.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 2037774,
|
|
"checkpoint_bytes": 8156921
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 8156921
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 0.9690986667313458,
|
|
"single_query_latency_ms_p50": 0.901999999769032,
|
|
"single_query_latency_ms_p95": 1.4788999978918582,
|
|
"queries_per_second": 1031.8866740090364,
|
|
"samples": 150,
|
|
"batched_qps_64": 77868.35385475321,
|
|
"batched_latency_ms_64": 0.012842187493333768,
|
|
"batched_qps_256": 275832.3454737125,
|
|
"batched_latency_ms_256": 0.003625390627348679
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.37916733340049785,
|
|
"single_query_latency_ms_p50": 0.37180000072112307,
|
|
"single_query_latency_ms_p95": 0.5995000028633513,
|
|
"queries_per_second": 2637.3579997824963,
|
|
"samples": 150,
|
|
"batched_qps_64": 12773.737196355747,
|
|
"batched_latency_ms_64": 0.07828562499980762,
|
|
"batched_qps_256": 12928.000129629021,
|
|
"batched_latency_ms_256": 0.07735148437291173
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.9436959694622207,
|
|
"recall1": 0.8882059054676097,
|
|
"recall3": 0.9639609296059279,
|
|
"recall5": 0.973981138430448,
|
|
"recall8": 0.9864151790726395,
|
|
"evidence_count_in_top3": 1.0717974626698104,
|
|
"hit3": 0.9683956438756035,
|
|
"mrr": 0.9597054333626427,
|
|
"ndcg3": 0.9530820526370456,
|
|
"mean_first_positive_rank": 1.2573818345121814,
|
|
"all_evidence_in_top3": 0.9595262153362524,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.921,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 51.315328869764656,
|
|
"median_score_margin": 56.234654903411865,
|
|
"negative_margin_rate": 0.05630403053777928,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999984274064,
|
|
"unknown_mean_need_prob": 4.836221294362452e-06,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 4.8406160974828055e-06,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.9838125,
|
|
"recall1": 0.92203125,
|
|
"recall3": 0.9950625,
|
|
"recall5": 0.99978125,
|
|
"recall8": 0.999875,
|
|
"evidence_count_in_top3": 1.115125,
|
|
"hit3": 1.0,
|
|
"mrr": 0.991875,
|
|
"ndcg3": 0.9876817889701721,
|
|
"mean_first_positive_rank": 1.016375,
|
|
"all_evidence_in_top3": 0.990125,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.921,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 57.110110210259094,
|
|
"median_score_margin": 59.90444374084473,
|
|
"negative_margin_rate": 0.0161875,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999999701977,
|
|
"unknown_mean_need_prob": 6.379481702447893e-11,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.5940246045694201,
|
|
"recall1": 0.5940246045694201,
|
|
"recall3": 0.6842413591095489,
|
|
"recall5": 0.7352079671939075,
|
|
"recall8": 0.8611599297012302,
|
|
"evidence_count_in_top3": 0.6842413591095489,
|
|
"hit3": 0.6842413591095489,
|
|
"mrr": 0.6760273585431921,
|
|
"ndcg3": 0.6466497456649108,
|
|
"mean_first_positive_rank": 3.4463971880492092,
|
|
"all_evidence_in_top3": 0.6842413591095489,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.27154229221702947,
|
|
"median_score_margin": 7.975786209106445,
|
|
"negative_margin_rate": 0.40597539543058,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9999999838679872,
|
|
"unknown_mean_need_prob": 0.0004654159529796483,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.5233644859813084,
|
|
"recall1": 0.5233644859813084,
|
|
"recall3": 0.7757009345794392,
|
|
"recall5": 0.9252336448598131,
|
|
"recall8": 0.9719626168224299,
|
|
"evidence_count_in_top3": 0.7757009345794392,
|
|
"hit3": 0.7757009345794392,
|
|
"mrr": 0.674896157840083,
|
|
"ndcg3": 0.6678873486315127,
|
|
"mean_first_positive_rank": 2.3738317757009346,
|
|
"all_evidence_in_top3": 0.7757009345794392,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.8786688875929217,
|
|
"median_score_margin": 0.4756507873535156,
|
|
"negative_margin_rate": 0.4766355140186916,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0004654159529796483,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.993,
|
|
"recall1": 0.993,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9965,
|
|
"ndcg3": 0.9974165082750002,
|
|
"mean_first_positive_rank": 1.007,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 54.4210530708991,
|
|
"median_score_margin": 56.92102670669556,
|
|
"negative_margin_rate": 0.007,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.975,
|
|
"recall1": 0.975,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9875,
|
|
"ndcg3": 0.9907732438392864,
|
|
"mean_first_positive_rank": 1.025,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 64.29146421498805,
|
|
"median_score_margin": 73.40720510482788,
|
|
"negative_margin_rate": 0.025,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9845,
|
|
"recall1": 0.9845,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99225,
|
|
"ndcg3": 0.9942794111803576,
|
|
"mean_first_positive_rank": 1.0155,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 65.52974343589041,
|
|
"median_score_margin": 73.97303533554077,
|
|
"negative_margin_rate": 0.0155,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 1.275862421441728e-10,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9795,
|
|
"recall1": 0.9795,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9896666666666666,
|
|
"ndcg3": 0.9923685950714292,
|
|
"mean_first_positive_rank": 1.021,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 60.150181606799364,
|
|
"median_score_margin": 68.01980018615723,
|
|
"negative_margin_rate": 0.0205,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9885,
|
|
"recall1": 0.49425,
|
|
"recall3": 0.9605,
|
|
"recall5": 0.99825,
|
|
"recall8": 0.999,
|
|
"evidence_count_in_top3": 1.921,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99425,
|
|
"ndcg3": 0.9452009954703022,
|
|
"mean_first_positive_rank": 1.0115,
|
|
"all_evidence_in_top3": 0.921,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.921,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 55.6439196386747,
|
|
"median_score_margin": 62.50904202461243,
|
|
"negative_margin_rate": 0.0115,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999997615814,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.995,
|
|
"recall1": 0.995,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9975,
|
|
"ndcg3": 0.9981546487678573,
|
|
"mean_first_positive_rank": 1.005,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 48.13904330630228,
|
|
"median_score_margin": 49.25835180282593,
|
|
"negative_margin_rate": 0.005,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9805,
|
|
"recall1": 0.9805,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9901666666666666,
|
|
"ndcg3": 0.9927376653178577,
|
|
"mean_first_positive_rank": 1.02,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 50.85701001610607,
|
|
"median_score_margin": 54.9089093208313,
|
|
"negative_margin_rate": 0.0195,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9745,
|
|
"recall1": 0.9745,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9871666666666666,
|
|
"ndcg3": 0.9905232438392865,
|
|
"mean_first_positive_rank": 1.026,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 57.84846639241278,
|
|
"median_score_margin": 63.093711376190186,
|
|
"negative_margin_rate": 0.0255,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.5898566703417861,
|
|
"recall1": 0.5898566703417861,
|
|
"recall3": 0.6896361631753032,
|
|
"recall5": 0.7464167585446527,
|
|
"recall8": 0.8676957001102535,
|
|
"evidence_count_in_top3": 0.6896361631753032,
|
|
"hit3": 0.6896361631753032,
|
|
"mrr": 0.6759606339151697,
|
|
"ndcg3": 0.6479024598421029,
|
|
"mean_first_positive_rank": 3.383131201764057,
|
|
"all_evidence_in_top3": 0.6896361631753032,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.20369631854576997,
|
|
"median_score_margin": 6.827788352966309,
|
|
"negative_margin_rate": 0.4101433296582139,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9999999848195447,
|
|
"unknown_mean_need_prob": 0.00018733273071117103,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 3.3919047850488803e-15,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"ZOV-128 final": {
|
|
"checkpoint": "checkpoints\\router_zov_v2_128\\memory_router_v2.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 2037774,
|
|
"checkpoint_bytes": 8156921
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 8156921
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 1.0321086667924344,
|
|
"single_query_latency_ms_p50": 0.9837000034167431,
|
|
"single_query_latency_ms_p95": 1.4700000028824434,
|
|
"queries_per_second": 968.8902265570339,
|
|
"samples": 150,
|
|
"batched_qps_64": 63339.997213904906,
|
|
"batched_latency_ms_64": 0.015787812503731402,
|
|
"batched_qps_256": 251226.69281453046,
|
|
"batched_latency_ms_256": 0.00398046875034197
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.395059999912822,
|
|
"single_query_latency_ms_p50": 0.38700000004610047,
|
|
"single_query_latency_ms_p95": 0.5204000044614077,
|
|
"queries_per_second": 2531.2610748257744,
|
|
"samples": 150,
|
|
"batched_qps_64": 13292.67988538735,
|
|
"batched_latency_ms_64": 0.07522937501107663,
|
|
"batched_qps_256": 13652.896440499333,
|
|
"batched_latency_ms_256": 0.07324453125079344
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.9071516784551477,
|
|
"recall1": 0.8525317166273717,
|
|
"recall3": 0.9369035589985405,
|
|
"recall5": 0.9511620074099023,
|
|
"recall8": 0.9639047939822611,
|
|
"evidence_count_in_top3": 1.0318288986190636,
|
|
"hit3": 0.9542494667115752,
|
|
"mrr": 0.9350004764720156,
|
|
"ndcg3": 0.921200829570768,
|
|
"mean_first_positive_rank": 1.555349724935444,
|
|
"all_evidence_in_top3": 0.9195576512855058,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.691,
|
|
"hop_accuracy": 0.9158759124087591,
|
|
"hop_accuracy_answerable": 0.8964859099584597,
|
|
"hop_under_prediction": 0.10351409004154036,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 24.21305508213604,
|
|
"median_score_margin": 25.759035110473633,
|
|
"negative_margin_rate": 0.09284832154485237,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 1844,
|
|
"true_1_pred_1": 13970,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.868957594983749,
|
|
"unknown_mean_need_prob": 1.5445844613853507e-10,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9169754125968339,
|
|
"need_f1": 0.956689800579812,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9325273722627737,
|
|
"known_question_refusal_rate": 0.08302458740316605,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16335,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 1479
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8759964073200853,
|
|
"need_f1": 0.9338998773152997,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.8992244525547445,
|
|
"known_question_refusal_rate": 0.12400359267991468,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 15605,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 2209
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8615134164140563,
|
|
"need_f1": 0.9256053798136364,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.8874543795620438,
|
|
"known_question_refusal_rate": 0.13848658358594365,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 15347,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 2467
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8427079824856855,
|
|
"need_f1": 0.9146408334856516,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.8721715328467153,
|
|
"known_question_refusal_rate": 0.1572920175143146,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 15012,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 2802
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8322667564836645,
|
|
"need_f1": 0.9084558823529412,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.8636861313868613,
|
|
"known_question_refusal_rate": 0.16773324351633548,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 14826,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 2988
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8274952284719883,
|
|
"need_f1": 0.9056058977115651,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.859808394160584,
|
|
"known_question_refusal_rate": 0.17250477152801166,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 14741,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 3073
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 3.3175517012669046e-11,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.956125,
|
|
"recall1": 0.8953125,
|
|
"recall3": 0.9751875,
|
|
"recall5": 0.9816875,
|
|
"recall8": 0.98475,
|
|
"evidence_count_in_top3": 1.080875,
|
|
"hit3": 0.9945,
|
|
"mrr": 0.9756561115666375,
|
|
"ndcg3": 0.9636228207179964,
|
|
"mean_first_positive_rank": 1.0985,
|
|
"all_evidence_in_top3": 0.955875,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.691,
|
|
"hop_accuracy": 0.92095,
|
|
"hop_accuracy_answerable": 0.9011875,
|
|
"hop_under_prediction": 0.0988125,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 27.2492798316353,
|
|
"median_score_margin": 28.251808643341064,
|
|
"negative_margin_rate": 0.043875,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 1581,
|
|
"true_1_pred_1": 12419,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.8787204854432493,
|
|
"unknown_mean_need_prob": 3.7716760582211954e-16,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8618125,
|
|
"need_f1": 0.9257779717345328,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.88945,
|
|
"known_question_refusal_rate": 0.1381875,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 13789,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 2211
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.4967779730521383,
|
|
"recall1": 0.4967779730521383,
|
|
"recall3": 0.6121851200937317,
|
|
"recall5": 0.6877562975981254,
|
|
"recall8": 0.7773872290568249,
|
|
"evidence_count_in_top3": 0.6121851200937317,
|
|
"hit3": 0.6121851200937317,
|
|
"mrr": 0.5919114414798028,
|
|
"ndcg3": 0.56399255386225,
|
|
"mean_first_positive_rank": 5.587580550673697,
|
|
"all_evidence_in_top3": 0.6121851200937317,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.8478009259259259,
|
|
"hop_accuracy_answerable": 0.8459285295840656,
|
|
"hop_under_prediction": 0.1540714704159344,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -2.0129484753023483,
|
|
"median_score_margin": -0.20529747009277344,
|
|
"negative_margin_rate": 0.5032220269478618,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 263,
|
|
"true_1_pred_1": 1444
|
|
},
|
|
"known_mean_need_prob": 0.7692814919056603,
|
|
"unknown_mean_need_prob": 1.5049562811561687e-08,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8500292911540714,
|
|
"need_f1": 0.9189360354654844,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.8518518518518519,
|
|
"known_question_refusal_rate": 0.14997070884592853,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1451,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 256
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.1308411214953271,
|
|
"recall1": 0.1308411214953271,
|
|
"recall3": 0.3925233644859813,
|
|
"recall5": 0.5887850467289719,
|
|
"recall8": 0.822429906542056,
|
|
"evidence_count_in_top3": 0.3925233644859813,
|
|
"hit3": 0.3925233644859813,
|
|
"mrr": 0.3290455345819019,
|
|
"ndcg3": 0.2763659536715653,
|
|
"mean_first_positive_rank": 5.542056074766355,
|
|
"all_evidence_in_top3": 0.3925233644859813,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -11.411317996750368,
|
|
"median_score_margin": -10.944583892822266,
|
|
"negative_margin_rate": 0.8691588785046729,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 0.9992460118275936,
|
|
"unknown_mean_need_prob": 1.5049562811561687e-08,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.968,
|
|
"recall1": 0.968,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9838333333333333,
|
|
"ndcg3": 0.9880588223607152,
|
|
"mean_first_positive_rank": 1.033,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.509,
|
|
"hop_accuracy_answerable": 0.509,
|
|
"hop_under_prediction": 0.491,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 26.64511842997372,
|
|
"median_score_margin": 26.41234827041626,
|
|
"negative_margin_rate": 0.032,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 982,
|
|
"true_1_pred_1": 1018
|
|
},
|
|
"known_mean_need_prob": 0.3932232758104801,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.239,
|
|
"need_f1": 0.3857949959644875,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 0.239,
|
|
"known_question_refusal_rate": 0.761,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 478,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 1522
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.948,
|
|
"recall1": 0.948,
|
|
"recall3": 0.9995,
|
|
"recall5": 0.9995,
|
|
"recall8": 0.9995,
|
|
"evidence_count_in_top3": 0.9995,
|
|
"hit3": 0.9995,
|
|
"mrr": 0.9736851851851852,
|
|
"ndcg3": 0.9804274174321443,
|
|
"mean_first_positive_rank": 1.065,
|
|
"all_evidence_in_top3": 0.9995,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 25.91951792217791,
|
|
"median_score_margin": 24.046737670898438,
|
|
"negative_margin_rate": 0.052,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9997872362732887,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9575,
|
|
"recall1": 0.9575,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9785833333333334,
|
|
"ndcg3": 0.9841835847732154,
|
|
"mean_first_positive_rank": 1.0435,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 25.73169531854242,
|
|
"median_score_margin": 24.141990661621094,
|
|
"negative_margin_rate": 0.0425,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9997921265959739,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 7.538130395219147e-16,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.954,
|
|
"recall1": 0.954,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9768333333333333,
|
|
"ndcg3": 0.9828918389107156,
|
|
"mean_first_positive_rank": 1.047,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 24.1671119158864,
|
|
"median_score_margin": 20.684181213378906,
|
|
"negative_margin_rate": 0.046,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9998738940358162,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.973,
|
|
"recall1": 0.4865,
|
|
"recall3": 0.8455,
|
|
"recall5": 0.886,
|
|
"recall8": 0.903,
|
|
"evidence_count_in_top3": 1.691,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9864166666666666,
|
|
"ndcg3": 0.8584171108011072,
|
|
"mean_first_positive_rank": 1.0275,
|
|
"all_evidence_in_top3": 0.691,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.691,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 31.524886593375356,
|
|
"median_score_margin": 34.437350273132324,
|
|
"negative_margin_rate": 0.027,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9990347371697426,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.982,
|
|
"recall1": 0.982,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9909166666666667,
|
|
"ndcg3": 0.9932912706875006,
|
|
"mean_first_positive_rank": 1.0185,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.8295,
|
|
"hop_accuracy_answerable": 0.8295,
|
|
"hop_under_prediction": 0.1705,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 29.277884820342063,
|
|
"median_score_margin": 29.79875612258911,
|
|
"negative_margin_rate": 0.018,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 341,
|
|
"true_1_pred_1": 1659
|
|
},
|
|
"known_mean_need_prob": 0.7633418458849192,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.7845,
|
|
"need_f1": 0.8792378817595965,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 0.7845,
|
|
"known_question_refusal_rate": 0.2155,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 1569,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 431
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9035,
|
|
"recall1": 0.9035,
|
|
"recall3": 0.9565,
|
|
"recall5": 0.968,
|
|
"recall8": 0.9755,
|
|
"evidence_count_in_top3": 0.9565,
|
|
"hit3": 0.9565,
|
|
"mrr": 0.9337303740145813,
|
|
"ndcg3": 0.935564514526787,
|
|
"mean_first_positive_rank": 1.515,
|
|
"all_evidence_in_top3": 0.9565,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.871,
|
|
"hop_accuracy_answerable": 0.871,
|
|
"hop_under_prediction": 0.129,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 22.111119223967194,
|
|
"median_score_margin": 23.948859691619873,
|
|
"negative_margin_rate": 0.0965,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 258,
|
|
"true_1_pred_1": 1742
|
|
},
|
|
"known_mean_need_prob": 0.8747201321423054,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.871,
|
|
"need_f1": 0.9310529128808124,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 0.871,
|
|
"known_question_refusal_rate": 0.129,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 1742,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 258
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.963,
|
|
"recall1": 0.963,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98125,
|
|
"ndcg3": 0.9861480062517868,
|
|
"mean_first_positive_rank": 1.0385,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 32.616904428817335,
|
|
"median_score_margin": 31.700976371765137,
|
|
"negative_margin_rate": 0.037,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999906356334687,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.47519294377067256,
|
|
"recall1": 0.47519294377067256,
|
|
"recall3": 0.5992282249173098,
|
|
"recall5": 0.6819184123484013,
|
|
"recall8": 0.7800441014332966,
|
|
"evidence_count_in_top3": 0.5992282249173098,
|
|
"hit3": 0.5992282249173098,
|
|
"mrr": 0.5764061206208859,
|
|
"ndcg3": 0.5470267069932294,
|
|
"mean_first_positive_rank": 5.58489525909592,
|
|
"all_evidence_in_top3": 0.5992282249173098,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.8630208333333333,
|
|
"hop_accuracy_answerable": 0.8550165380374862,
|
|
"hop_under_prediction": 0.14498346196251377,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -2.5673175705586537,
|
|
"median_score_margin": -1.374494194984436,
|
|
"negative_margin_rate": 0.5248070562293274,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 263,
|
|
"true_1_pred_1": 1551
|
|
},
|
|
"known_mean_need_prob": 0.7828461025074502,
|
|
"unknown_mean_need_prob": 5.983064822400017e-09,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8588754134509372,
|
|
"need_f1": 0.9240806642941874,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.8666666666666667,
|
|
"known_question_refusal_rate": 0.14112458654906285,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1558,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 256
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 5.221721223244015e-19,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} |