WpyQwq
643e22ecb9
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线 - 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同） - 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00） - 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00% - 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益） - 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
4358 lines
138 KiB
JSON
4358 lines
138 KiB
JSON
{
|
|
"V2-128 deployed(v3)": {
|
|
"checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_qwen_router_entities\\memory_router_v2.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 2037774,
|
|
"checkpoint_bytes": 8157561
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 8157561
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 0.9698253340320662,
|
|
"single_query_latency_ms_p50": 0.8987999899545684,
|
|
"single_query_latency_ms_p95": 1.46159999712836,
|
|
"queries_per_second": 1031.1135056067076,
|
|
"samples": 150,
|
|
"batched_qps_64": 64355.240952926746,
|
|
"batched_latency_ms_64": 0.015538749994448153,
|
|
"batched_qps_256": 185281.68609572947,
|
|
"batched_latency_ms_256": 0.005397187499056599
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.47286199949060875,
|
|
"single_query_latency_ms_p50": 0.4398000019136816,
|
|
"single_query_latency_ms_p95": 0.7371999963652343,
|
|
"queries_per_second": 2114.7819048205424,
|
|
"samples": 150,
|
|
"batched_qps_64": 11860.681466374921,
|
|
"batched_latency_ms_64": 0.08431218752775749,
|
|
"batched_qps_256": 12256.36876291884,
|
|
"batched_latency_ms_256": 0.08159023437883661
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.4112495789828225,
|
|
"recall1": 0.37027057370607386,
|
|
"recall3": 0.4673009992141013,
|
|
"recall5": 0.4891377568204783,
|
|
"recall8": 0.5160267205568654,
|
|
"evidence_count_in_top3": 0.5420455821264174,
|
|
"hit3": 0.5003368137420007,
|
|
"mrr": 0.476908110965675,
|
|
"ndcg3": 0.44220238482445295,
|
|
"mean_first_positive_rank": 14.38671831144044,
|
|
"all_evidence_in_top3": 0.43426518468620184,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.3715,
|
|
"hop_accuracy": 0.7322992700729927,
|
|
"hop_accuracy_answerable": 0.9010890310991355,
|
|
"hop_under_prediction": 0.046817110138093636,
|
|
"hop_over_prediction": 0.05209385876277085,
|
|
"mean_score_margin": -9.29892828391031,
|
|
"median_score_margin": -9.269989609718323,
|
|
"negative_margin_rate": 0.5887504210171774,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 4042,
|
|
"true_0_pred_2": 64,
|
|
"true_1_pred_0": 834,
|
|
"true_1_pred_1": 14052,
|
|
"true_1_pred_2": 928,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9762653562039442,
|
|
"unknown_mean_need_prob": 0.998855464207621,
|
|
"thr0.30": {
|
|
"need_precision": 0.8126824817518248,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.8966628076710123,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8126824817518248,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.8126824817518248,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.8966628076710123,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8126824817518248,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.8124086257309941,
|
|
"need_recall": 0.998203660042663,
|
|
"need_f1": 0.895773512669387,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8112226277372263,
|
|
"known_question_refusal_rate": 0.001796339957336926,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17782,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 32
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.8115476409032495,
|
|
"need_recall": 0.9925900976759852,
|
|
"need_f1": 0.892985202767537,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8066605839416059,
|
|
"known_question_refusal_rate": 0.00740990232401482,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17682,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 132
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 0.8112270700197692,
|
|
"need_recall": 0.9905130796003143,
|
|
"need_f1": 0.8919499557689878,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8049726277372263,
|
|
"known_question_refusal_rate": 0.00948692039968564,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17645,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 169
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 0.8106525247867189,
|
|
"need_recall": 0.9868081284383069,
|
|
"need_f1": 0.8900984835059115,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8019616788321168,
|
|
"known_question_refusal_rate": 0.01319187156169305,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17579,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 235
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_2": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.9999341862276196,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 64,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.442375,
|
|
"recall1": 0.39675,
|
|
"recall3": 0.49153125,
|
|
"recall5": 0.50303125,
|
|
"recall8": 0.51234375,
|
|
"evidence_count_in_top3": 0.57475,
|
|
"hit3": 0.5283125,
|
|
"mrr": 0.500402578720693,
|
|
"ndcg3": 0.4692792057097287,
|
|
"mean_first_positive_rank": 14.9525,
|
|
"all_evidence_in_top3": 0.45475,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.3715,
|
|
"hop_accuracy": 0.74955,
|
|
"hop_accuracy_answerable": 0.9369375,
|
|
"hop_under_prediction": 0.0160625,
|
|
"hop_over_prediction": 0.047,
|
|
"mean_score_margin": -8.091850390457548,
|
|
"median_score_margin": -6.662378132343292,
|
|
"negative_margin_rate": 0.557625,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 4000,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 257,
|
|
"true_1_pred_1": 12991,
|
|
"true_1_pred_2": 752,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9821553228273988,
|
|
"unknown_mean_need_prob": 0.9988265322595835,
|
|
"thr0.50": {
|
|
"need_precision": 0.8,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.888888888888889,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 0,
|
|
"fp": 4000,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.14528412419449327,
|
|
"recall1": 0.14528412419449327,
|
|
"recall3": 0.2677211482132396,
|
|
"recall5": 0.38488576449912126,
|
|
"recall8": 0.5653192735793791,
|
|
"evidence_count_in_top3": 0.2677211482132396,
|
|
"hit3": 0.2677211482132396,
|
|
"mrr": 0.2801716446629072,
|
|
"ndcg3": 0.21516992510461316,
|
|
"mean_first_positive_rank": 9.195664909197422,
|
|
"all_evidence_in_top3": 0.2677211482132396,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.5740740740740741,
|
|
"hop_accuracy_answerable": 0.5811364967779731,
|
|
"hop_under_prediction": 0.3380199179847686,
|
|
"hop_over_prediction": 0.08084358523725835,
|
|
"mean_score_margin": -19.359866383266713,
|
|
"median_score_margin": -18.159542322158813,
|
|
"negative_margin_rate": 0.8547158758055068,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 21,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 577,
|
|
"true_1_pred_1": 992,
|
|
"true_1_pred_2": 138
|
|
},
|
|
"known_mean_need_prob": 0.9195722371566764,
|
|
"unknown_mean_need_prob": 0.9999671209426153,
|
|
"thr0.50": {
|
|
"need_precision": 0.9876179245283019,
|
|
"need_recall": 0.981253661394259,
|
|
"need_f1": 0.9844255069056715,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.9693287037037037,
|
|
"known_question_refusal_rate": 0.018746338605741066,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 1675,
|
|
"tn": 0,
|
|
"fp": 21,
|
|
"fn": 32
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.028037383177570093,
|
|
"recall5": 0.07476635514018691,
|
|
"recall8": 0.2803738317757009,
|
|
"evidence_count_in_top3": 0.028037383177570093,
|
|
"hit3": 0.028037383177570093,
|
|
"mrr": 0.1023068389893556,
|
|
"ndcg3": 0.01524233414552764,
|
|
"mean_first_positive_rank": 12.598130841121495,
|
|
"all_evidence_in_top3": 0.028037383177570093,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.5390625,
|
|
"hop_accuracy_answerable": 0.6448598130841121,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.35514018691588783,
|
|
"mean_score_margin": -29.291684916086286,
|
|
"median_score_margin": -29.07153034210205,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 21,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 69,
|
|
"true_1_pred_2": 38
|
|
},
|
|
"known_mean_need_prob": 0.9999633771236812,
|
|
"unknown_mean_need_prob": 0.9999671209426153,
|
|
"thr0.50": {
|
|
"need_precision": 0.8359375,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9106382978723404,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8359375,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 0,
|
|
"fp": 21,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.796,
|
|
"recall1": 0.796,
|
|
"recall3": 0.9125,
|
|
"recall5": 0.919,
|
|
"recall8": 0.927,
|
|
"evidence_count_in_top3": 0.9125,
|
|
"hit3": 0.9125,
|
|
"mrr": 0.8582544951434297,
|
|
"ndcg3": 0.8678012294946459,
|
|
"mean_first_positive_rank": 2.8275,
|
|
"all_evidence_in_top3": 0.9125,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.4955,
|
|
"hop_accuracy_answerable": 0.4955,
|
|
"hop_under_prediction": 0.1285,
|
|
"hop_over_prediction": 0.376,
|
|
"mean_score_margin": 21.036133953779935,
|
|
"median_score_margin": 21.400100469589233,
|
|
"negative_margin_rate": 0.204,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 257,
|
|
"true_1_pred_1": 991,
|
|
"true_1_pred_2": 752
|
|
},
|
|
"known_mean_need_prob": 0.9764632563889026,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.231,
|
|
"recall1": 0.231,
|
|
"recall3": 0.2535,
|
|
"recall5": 0.255,
|
|
"recall8": 0.257,
|
|
"evidence_count_in_top3": 0.2535,
|
|
"hit3": 0.2535,
|
|
"mrr": 0.26603145346142226,
|
|
"ndcg3": 0.24486859507142916,
|
|
"mean_first_positive_rank": 23.789,
|
|
"all_evidence_in_top3": 0.2535,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -27.718347060799598,
|
|
"median_score_margin": -38.59446668624878,
|
|
"negative_margin_rate": 0.769,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9723067943453789,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.2255,
|
|
"recall1": 0.2255,
|
|
"recall3": 0.2455,
|
|
"recall5": 0.2475,
|
|
"recall8": 0.2485,
|
|
"evidence_count_in_top3": 0.2455,
|
|
"hit3": 0.2455,
|
|
"mrr": 0.2596338049221357,
|
|
"ndcg3": 0.23792220044107196,
|
|
"mean_first_positive_rank": 24.0695,
|
|
"all_evidence_in_top3": 0.2455,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -28.903205164782705,
|
|
"median_score_margin": -39.01692509651184,
|
|
"negative_margin_rate": 0.7745,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9724746745526791,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.99774328109622,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 2000,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5255,
|
|
"recall1": 0.5255,
|
|
"recall3": 0.6245,
|
|
"recall5": 0.6265,
|
|
"recall8": 0.6295,
|
|
"evidence_count_in_top3": 0.6245,
|
|
"hit3": 0.6245,
|
|
"mrr": 0.5855216777342396,
|
|
"ndcg3": 0.585867169546431,
|
|
"mean_first_positive_rank": 12.2085,
|
|
"all_evidence_in_top3": 0.6245,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -1.0039909677058458,
|
|
"median_score_margin": 2.929476022720337,
|
|
"negative_margin_rate": 0.4745,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9875293919146061,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.73,
|
|
"recall1": 0.365,
|
|
"recall3": 0.66575,
|
|
"recall5": 0.72725,
|
|
"recall8": 0.77375,
|
|
"evidence_count_in_top3": 1.3315,
|
|
"hit3": 0.96,
|
|
"mrr": 0.84532448592014,
|
|
"ndcg3": 0.6658792959224611,
|
|
"mean_first_positive_rank": 1.59,
|
|
"all_evidence_in_top3": 0.3715,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.3715,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 2.786984693795443,
|
|
"median_score_margin": 5.27826452255249,
|
|
"negative_margin_rate": 0.27,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9993210740983486,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.443,
|
|
"recall1": 0.443,
|
|
"recall3": 0.5525,
|
|
"recall5": 0.5615,
|
|
"recall8": 0.5665,
|
|
"evidence_count_in_top3": 0.5525,
|
|
"hit3": 0.5525,
|
|
"mrr": 0.5118924660255056,
|
|
"ndcg3": 0.5092063534375025,
|
|
"mean_first_positive_rank": 13.8255,
|
|
"all_evidence_in_top3": 0.5525,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -2.995655853047967,
|
|
"median_score_margin": -5.5251545906066895,
|
|
"negative_margin_rate": 0.557,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9880377069413662,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.249,
|
|
"recall1": 0.249,
|
|
"recall3": 0.287,
|
|
"recall5": 0.293,
|
|
"recall8": 0.3005,
|
|
"evidence_count_in_top3": 0.287,
|
|
"hit3": 0.287,
|
|
"mrr": 0.2919631936667751,
|
|
"ndcg3": 0.27173149797678653,
|
|
"mean_first_positive_rank": 22.067,
|
|
"all_evidence_in_top3": 0.287,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -19.72415616109967,
|
|
"median_score_margin": -21.928287506103516,
|
|
"negative_margin_rate": 0.751,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9656237106621265,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.339,
|
|
"recall1": 0.339,
|
|
"recall3": 0.391,
|
|
"recall5": 0.3945,
|
|
"recall8": 0.396,
|
|
"evidence_count_in_top3": 0.391,
|
|
"hit3": 0.391,
|
|
"mrr": 0.3845990528918967,
|
|
"ndcg3": 0.3709573037875013,
|
|
"mean_first_positive_rank": 19.243,
|
|
"all_evidence_in_top3": 0.391,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -8.212566563799978,
|
|
"median_score_margin": -16.937832549214363,
|
|
"negative_margin_rate": 0.661,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9954859737157822,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.13671444321940462,
|
|
"recall1": 0.13671444321940462,
|
|
"recall3": 0.2535832414553473,
|
|
"recall5": 0.36659316427783906,
|
|
"recall8": 0.5485115766262404,
|
|
"evidence_count_in_top3": 0.2535832414553473,
|
|
"hit3": 0.2535832414553473,
|
|
"mrr": 0.26968017045834825,
|
|
"ndcg3": 0.2033770627933551,
|
|
"mean_first_positive_rank": 9.396361631753033,
|
|
"all_evidence_in_top3": 0.2535832414553473,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.5526041666666667,
|
|
"hop_accuracy_answerable": 0.5848952590959207,
|
|
"hop_under_prediction": 0.31808158765159866,
|
|
"hop_over_prediction": 0.0970231532524807,
|
|
"mean_score_margin": -19.945701324287494,
|
|
"median_score_margin": -18.862584590911865,
|
|
"negative_margin_rate": 0.8632855567805954,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 42,
|
|
"true_0_pred_2": 64,
|
|
"true_1_pred_0": 577,
|
|
"true_1_pred_1": 1061,
|
|
"true_1_pred_2": 176
|
|
},
|
|
"known_mean_need_prob": 0.9243141621712682,
|
|
"unknown_mean_need_prob": 0.9999472358316746,
|
|
"thr0.50": {
|
|
"need_precision": 0.9438559322033898,
|
|
"need_recall": 0.9823594266813671,
|
|
"need_f1": 0.9627228525121555,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.928125,
|
|
"known_question_refusal_rate": 0.017640573318632856,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 1782,
|
|
"tn": 0,
|
|
"fp": 106,
|
|
"fn": 32
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.9999097834229469,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 2000,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"REPLAY-128 v7": {
|
|
"checkpoint": "checkpoints\\router_replay_v7_v2_128\\memory_router_v2.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 2037774,
|
|
"checkpoint_bytes": 8156921
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 8156921
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 0.9359393330911795,
|
|
"single_query_latency_ms_p50": 0.8711999980732799,
|
|
"single_query_latency_ms_p95": 1.5259000065270811,
|
|
"queries_per_second": 1068.445319738026,
|
|
"samples": 150,
|
|
"batched_qps_64": 65927.72676538704,
|
|
"batched_latency_ms_64": 0.015168124991760124,
|
|
"batched_qps_256": 262171.5169506191,
|
|
"batched_latency_ms_256": 0.003814296883319912
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.40274533355841413,
|
|
"single_query_latency_ms_p50": 0.36700000055134296,
|
|
"single_query_latency_ms_p95": 0.6454000103985891,
|
|
"queries_per_second": 2482.9586259004045,
|
|
"samples": 150,
|
|
"batched_qps_64": 12091.395836736374,
|
|
"batched_latency_ms_64": 0.08270343751064502,
|
|
"batched_qps_256": 12591.79467573841,
|
|
"batched_latency_ms_256": 0.07941679687064607
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.9413944088918828,
|
|
"recall1": 0.8857920736499383,
|
|
"recall3": 0.9647748961490962,
|
|
"recall5": 0.9714550353654429,
|
|
"recall8": 0.9804086673402942,
|
|
"evidence_count_in_top3": 1.0744919726058157,
|
|
"hit3": 0.9673290670259347,
|
|
"mrr": 0.957689277532011,
|
|
"ndcg3": 0.9536787031890833,
|
|
"mean_first_positive_rank": 1.34001347254968,
|
|
"all_evidence_in_top3": 0.9622207252722578,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9545,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 40.14557690149532,
|
|
"median_score_margin": 45.34479379653931,
|
|
"negative_margin_rate": 0.05860559110811721,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999984274064,
|
|
"unknown_mean_need_prob": 4.836221294362452e-06,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 4.8406160974828055e-06,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.98225,
|
|
"recall1": 0.92034375,
|
|
"recall3": 0.99715625,
|
|
"recall5": 0.99959375,
|
|
"recall8": 0.9996875,
|
|
"evidence_count_in_top3": 1.1193125,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9911041666666667,
|
|
"ndcg3": 0.9893498205729669,
|
|
"mean_first_positive_rank": 1.017875,
|
|
"all_evidence_in_top3": 0.9943125,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9545,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 44.73340179644042,
|
|
"median_score_margin": 47.92497134208679,
|
|
"negative_margin_rate": 0.01775,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999999701977,
|
|
"unknown_mean_need_prob": 6.379481702447893e-11,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.5875805506736965,
|
|
"recall1": 0.5875805506736965,
|
|
"recall3": 0.6731107205623902,
|
|
"recall5": 0.7106033977738723,
|
|
"recall8": 0.7996485061511424,
|
|
"evidence_count_in_top3": 0.6731107205623902,
|
|
"hit3": 0.6731107205623902,
|
|
"mrr": 0.6633131807141506,
|
|
"ndcg3": 0.6380925044585338,
|
|
"mean_first_positive_rank": 4.295254833040421,
|
|
"all_evidence_in_top3": 0.6731107205623902,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.2721888081208264,
|
|
"median_score_margin": 8.391286849975586,
|
|
"negative_margin_rate": 0.4124194493263035,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9999999838679872,
|
|
"unknown_mean_need_prob": 0.0004654159529796483,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.4766355140186916,
|
|
"recall1": 0.4766355140186916,
|
|
"recall3": 0.7757009345794392,
|
|
"recall5": 0.9252336448598131,
|
|
"recall8": 0.9813084112149533,
|
|
"evidence_count_in_top3": 0.7757009345794392,
|
|
"hit3": 0.7757009345794392,
|
|
"mrr": 0.6573319982198488,
|
|
"ndcg3": 0.6543120031041451,
|
|
"mean_first_positive_rank": 2.364485981308411,
|
|
"all_evidence_in_top3": 0.7757009345794392,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -1.0887432181946586,
|
|
"median_score_margin": -0.6999244689941406,
|
|
"negative_margin_rate": 0.5233644859813084,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0004654159529796483,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.991,
|
|
"recall1": 0.991,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9955,
|
|
"ndcg3": 0.9966783677821431,
|
|
"mean_first_positive_rank": 1.009,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 42.24227243242413,
|
|
"median_score_margin": 42.77137851715088,
|
|
"negative_margin_rate": 0.009,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.973,
|
|
"recall1": 0.973,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9865,
|
|
"ndcg3": 0.9900351033464293,
|
|
"mean_first_positive_rank": 1.027,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 50.82861385926604,
|
|
"median_score_margin": 54.98523235321045,
|
|
"negative_margin_rate": 0.027,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.983,
|
|
"recall1": 0.983,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9915,
|
|
"ndcg3": 0.9937258058107148,
|
|
"mean_first_positive_rank": 1.017,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 51.79758185644448,
|
|
"median_score_margin": 55.65491724014282,
|
|
"negative_margin_rate": 0.017,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 1.275862421441728e-10,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.978,
|
|
"recall1": 0.978,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9889166666666667,
|
|
"ndcg3": 0.9918149897017863,
|
|
"mean_first_positive_rank": 1.0225,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 47.034401270009575,
|
|
"median_score_margin": 50.28937745094299,
|
|
"negative_margin_rate": 0.022,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9905,
|
|
"recall1": 0.49525,
|
|
"recall3": 0.97725,
|
|
"recall5": 0.99675,
|
|
"recall8": 0.9975,
|
|
"evidence_count_in_top3": 1.9545,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99525,
|
|
"ndcg3": 0.963831301989089,
|
|
"mean_first_positive_rank": 1.0095,
|
|
"all_evidence_in_top3": 0.9545,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9545,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 46.103771095309405,
|
|
"median_score_margin": 52.25391101837158,
|
|
"negative_margin_rate": 0.0095,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999997615814,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.994,
|
|
"recall1": 0.994,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.997,
|
|
"ndcg3": 0.9977855785214288,
|
|
"mean_first_positive_rank": 1.006,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 36.59333083114028,
|
|
"median_score_margin": 35.444119691848755,
|
|
"negative_margin_rate": 0.006,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.978,
|
|
"recall1": 0.978,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9889166666666667,
|
|
"ndcg3": 0.9918149897017863,
|
|
"mean_first_positive_rank": 1.0225,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 35.780669214252384,
|
|
"median_score_margin": 35.064393758773804,
|
|
"negative_margin_rate": 0.022,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9705,
|
|
"recall1": 0.9705,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98525,
|
|
"ndcg3": 0.989112427730358,
|
|
"mean_first_positive_rank": 1.0295,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 47.48657381267706,
|
|
"median_score_margin": 51.48151856660843,
|
|
"negative_margin_rate": 0.0295,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.5810363836824697,
|
|
"recall1": 0.5810363836824697,
|
|
"recall3": 0.679162072767365,
|
|
"recall5": 0.7232635060639471,
|
|
"recall8": 0.8103638368246968,
|
|
"evidence_count_in_top3": 0.679162072767365,
|
|
"hit3": 0.679162072767365,
|
|
"mrr": 0.662960376675071,
|
|
"ndcg3": 0.6390492224051051,
|
|
"mean_first_positive_rank": 4.181367144432194,
|
|
"all_evidence_in_top3": 0.679162072767365,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.32035381466873164,
|
|
"median_score_margin": 7.441610336303711,
|
|
"negative_margin_rate": 0.4189636163175303,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9999999848195447,
|
|
"unknown_mean_need_prob": 0.00018733273071117103,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 3.3919047850488803e-15,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"PROD-128 step20000": {
|
|
"checkpoint": "checkpoints\\router_prod_v2_128\\memory_router_v2.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 2037774,
|
|
"checkpoint_bytes": 8156921
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 8156921
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 0.9775833330544023,
|
|
"single_query_latency_ms_p50": 0.9395999950356781,
|
|
"single_query_latency_ms_p95": 1.3619999954244122,
|
|
"queries_per_second": 1022.9306967371857,
|
|
"samples": 150,
|
|
"batched_qps_64": 73335.62500742644,
|
|
"batched_latency_ms_64": 0.013635937511935481,
|
|
"batched_qps_256": 273837.7939855633,
|
|
"batched_latency_ms_256": 0.0036517968737825868
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.39715666566432145,
|
|
"single_query_latency_ms_p50": 0.3654000029200688,
|
|
"single_query_latency_ms_p95": 0.6416000105673447,
|
|
"queries_per_second": 2517.8980650552758,
|
|
"samples": 150,
|
|
"batched_qps_64": 11271.852422702024,
|
|
"batched_latency_ms_64": 0.08871656250448723,
|
|
"batched_qps_256": 13151.297458363848,
|
|
"batched_latency_ms_256": 0.07603812499610285
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.9378017289772089,
|
|
"recall1": 0.8834343774559336,
|
|
"recall3": 0.9658976086224318,
|
|
"recall5": 0.9732794431346132,
|
|
"recall8": 0.9842258897496351,
|
|
"evidence_count_in_top3": 1.0772426181654877,
|
|
"hit3": 0.9668238464129336,
|
|
"mrr": 0.9559179074946493,
|
|
"ndcg3": 0.9538607332074894,
|
|
"mean_first_positive_rank": 1.2769731671718874,
|
|
"all_evidence_in_top3": 0.96497137083193,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9835,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 21.892463866317613,
|
|
"median_score_margin": 25.104029178619385,
|
|
"negative_margin_rate": 0.06219827102279106,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999600001406399,
|
|
"unknown_mean_need_prob": 0.0011980682588774603,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.010227199368273432,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.9809375,
|
|
"recall1": 0.92040625,
|
|
"recall3": 0.99896875,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.1229375,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9904270833333334,
|
|
"ndcg3": 0.99126750245029,
|
|
"mean_first_positive_rank": 1.0193125,
|
|
"all_evidence_in_top3": 0.9979375,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9835,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 24.198241920885398,
|
|
"median_score_margin": 26.095373153686523,
|
|
"negative_margin_rate": 0.0190625,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999700157120824,
|
|
"unknown_mean_need_prob": 6.808238428402546e-05,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.5764499121265377,
|
|
"recall1": 0.5764499121265377,
|
|
"recall3": 0.6795547744581136,
|
|
"recall5": 0.7352079671939075,
|
|
"recall8": 0.8412419449326304,
|
|
"evidence_count_in_top3": 0.6795547744581136,
|
|
"hit3": 0.6795547744581136,
|
|
"mrr": 0.6631831932582257,
|
|
"ndcg3": 0.635135598759312,
|
|
"mean_first_positive_rank": 3.5219683655536027,
|
|
"all_evidence_in_top3": 0.6795547744581136,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 1.810913933616723,
|
|
"median_score_margin": 3.2161903381347656,
|
|
"negative_margin_rate": 0.4235500878734622,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9998666250922209,
|
|
"unknown_mean_need_prob": 0.09505709462488691,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.2523364485981308,
|
|
"recall1": 0.2523364485981308,
|
|
"recall3": 0.5887850467289719,
|
|
"recall5": 0.7757009345794392,
|
|
"recall8": 0.9065420560747663,
|
|
"evidence_count_in_top3": 0.5887850467289719,
|
|
"hit3": 0.5887850467289719,
|
|
"mrr": 0.4657435503229896,
|
|
"ndcg3": 0.44503359879840326,
|
|
"mean_first_positive_rank": 3.9906542056074765,
|
|
"all_evidence_in_top3": 0.5887850467289719,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -2.5303691987679384,
|
|
"median_score_margin": -1.9369163513183594,
|
|
"negative_margin_rate": 0.7476635514018691,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 0.9999519820525268,
|
|
"unknown_mean_need_prob": 0.09505709462488691,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.993,
|
|
"recall1": 0.993,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9965,
|
|
"ndcg3": 0.9974165082750002,
|
|
"mean_first_positive_rank": 1.007,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 26.333668621126563,
|
|
"median_score_margin": 27.148656368255615,
|
|
"negative_margin_rate": 0.007,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.999947308152914,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9755,
|
|
"recall1": 0.9755,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98775,
|
|
"ndcg3": 0.9909577789625007,
|
|
"mean_first_positive_rank": 1.0245,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 25.041911346442998,
|
|
"median_score_margin": 26.2372727394104,
|
|
"negative_margin_rate": 0.0245,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.999997846186161,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.983,
|
|
"recall1": 0.983,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9915,
|
|
"ndcg3": 0.9937258058107148,
|
|
"mean_first_positive_rank": 1.017,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 25.644431973803787,
|
|
"median_score_margin": 26.433398723602295,
|
|
"negative_margin_rate": 0.017,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999978821873665,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.00013525628542129198,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9845,
|
|
"recall1": 0.9845,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9921666666666666,
|
|
"ndcg3": 0.9942139463035718,
|
|
"mean_first_positive_rank": 1.016,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 23.74752949652076,
|
|
"median_score_margin": 26.17783832550049,
|
|
"negative_margin_rate": 0.0155,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999990122318267,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9685,
|
|
"recall1": 0.48425,
|
|
"recall3": 0.99175,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.9835,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9840833333333334,
|
|
"ndcg3": 0.9749284491737452,
|
|
"mean_first_positive_rank": 1.0325,
|
|
"all_evidence_in_top3": 0.9835,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9835,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 20.615374811934306,
|
|
"median_score_margin": 23.414374619722366,
|
|
"negative_margin_rate": 0.0315,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999314472973346,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9955,
|
|
"recall1": 0.9955,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99775,
|
|
"ndcg3": 0.9983391838910716,
|
|
"mean_first_positive_rank": 1.0045,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 22.83826958390698,
|
|
"median_score_margin": 24.362728118896484,
|
|
"negative_margin_rate": 0.0045,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999094858169556,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9815,
|
|
"recall1": 0.9815,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99075,
|
|
"ndcg3": 0.9931722004410719,
|
|
"mean_first_positive_rank": 1.0185,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 25.664584860108793,
|
|
"median_score_margin": 26.572662353515625,
|
|
"negative_margin_rate": 0.0185,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999773879647255,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.966,
|
|
"recall1": 0.966,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9829166666666667,
|
|
"ndcg3": 0.9873861467446439,
|
|
"mean_first_positive_rank": 1.0345,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 23.700164673238994,
|
|
"median_score_margin": 25.87112593650818,
|
|
"negative_margin_rate": 0.034,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.999999755859375,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.5573318632855567,
|
|
"recall1": 0.5573318632855567,
|
|
"recall3": 0.6742006615214995,
|
|
"recall5": 0.7375964718853363,
|
|
"recall8": 0.8450937155457552,
|
|
"evidence_count_in_top3": 0.6742006615214995,
|
|
"hit3": 0.6742006615214995,
|
|
"mrr": 0.6515370842207008,
|
|
"ndcg3": 0.6239223054870864,
|
|
"mean_first_positive_rank": 3.549614112458655,
|
|
"all_evidence_in_top3": 0.6742006615214995,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 1.5548404522687853,
|
|
"median_score_margin": 2.31545352935791,
|
|
"negative_margin_rate": 0.4426681367144432,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9998716599294606,
|
|
"unknown_mean_need_prob": 0.043839044658629714,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 9.084831467589538e-07,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"PROD-128 step10000": {
|
|
"checkpoint": "checkpoints\\router_prod_v2_128\\router_step_00010000.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": 10000,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 2037774,
|
|
"checkpoint_bytes": 24473667
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 24473667
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 1.2315193335719716,
|
|
"single_query_latency_ms_p50": 0.9520000021439046,
|
|
"single_query_latency_ms_p95": 1.6566999984206632,
|
|
"queries_per_second": 812.0051165575621,
|
|
"samples": 150,
|
|
"batched_qps_64": 49115.91353738528,
|
|
"batched_latency_ms_64": 0.020360000007713097,
|
|
"batched_qps_256": 141564.73262936738,
|
|
"batched_latency_ms_256": 0.007063906252824381
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.46193000006799895,
|
|
"single_query_latency_ms_p50": 0.3951999970013276,
|
|
"single_query_latency_ms_p95": 0.8545000018784776,
|
|
"queries_per_second": 2164.83016875456,
|
|
"samples": 150,
|
|
"batched_qps_64": 10985.200876163763,
|
|
"batched_latency_ms_64": 0.09103156248784217,
|
|
"batched_qps_256": 12158.516660730622,
|
|
"batched_latency_ms_256": 0.08224687500160144
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.9380824070955428,
|
|
"recall1": 0.8837711911979342,
|
|
"recall3": 0.9677500842034354,
|
|
"recall5": 0.9777702930279556,
|
|
"recall8": 0.9879869765353093,
|
|
"evidence_count_in_top3": 1.0793196362411586,
|
|
"hit3": 0.9684517794992702,
|
|
"mrr": 0.9570269878646712,
|
|
"ndcg3": 0.9552694474506009,
|
|
"mean_first_positive_rank": 1.2445829123161558,
|
|
"all_evidence_in_top3": 0.9670483889076008,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9875,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 18.42186796360512,
|
|
"median_score_margin": 18.455811977386475,
|
|
"negative_margin_rate": 0.06191759290445717,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999600001406399,
|
|
"unknown_mean_need_prob": 0.0011980682588774603,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.010227199368273432,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.9798125,
|
|
"recall1": 0.91934375,
|
|
"recall3": 0.99921875,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.1234375,
|
|
"hit3": 1.0,
|
|
"mrr": 0.989875,
|
|
"ndcg3": 0.991227295889911,
|
|
"mean_first_positive_rank": 1.020375,
|
|
"all_evidence_in_top3": 0.9984375,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9875,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 20.280915253964952,
|
|
"median_score_margin": 19.570133835077286,
|
|
"negative_margin_rate": 0.0201875,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999700157120824,
|
|
"unknown_mean_need_prob": 6.808238428402546e-05,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.5922671353251318,
|
|
"recall1": 0.5922671353251318,
|
|
"recall3": 0.700058582308143,
|
|
"recall5": 0.7844171060339777,
|
|
"recall8": 0.8816637375512595,
|
|
"evidence_count_in_top3": 0.700058582308143,
|
|
"hit3": 0.700058582308143,
|
|
"mrr": 0.6824865585659331,
|
|
"ndcg3": 0.6535262192986877,
|
|
"mean_first_positive_rank": 3.1646162858816638,
|
|
"all_evidence_in_top3": 0.700058582308143,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 2.295050391936091,
|
|
"median_score_margin": 2.6275386810302734,
|
|
"negative_margin_rate": 0.4077328646748682,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9998666250922209,
|
|
"unknown_mean_need_prob": 0.09505709462488691,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.21495327102803738,
|
|
"recall1": 0.21495327102803738,
|
|
"recall3": 0.5327102803738317,
|
|
"recall5": 0.7383177570093458,
|
|
"recall8": 0.8878504672897196,
|
|
"evidence_count_in_top3": 0.5327102803738317,
|
|
"hit3": 0.5327102803738317,
|
|
"mrr": 0.42499258270286305,
|
|
"ndcg3": 0.3921864140520735,
|
|
"mean_first_positive_rank": 4.140186915887851,
|
|
"all_evidence_in_top3": 0.5327102803738317,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -2.2910203627336805,
|
|
"median_score_margin": -1.6779632568359375,
|
|
"negative_margin_rate": 0.7850467289719626,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 0.9999519820525268,
|
|
"unknown_mean_need_prob": 0.09505709462488691,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9925,
|
|
"recall1": 0.9925,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99625,
|
|
"ndcg3": 0.9972319731517859,
|
|
"mean_first_positive_rank": 1.0075,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 20.927725780989974,
|
|
"median_score_margin": 18.922175407409668,
|
|
"negative_margin_rate": 0.0075,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.999947308152914,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.974,
|
|
"recall1": 0.974,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.987,
|
|
"ndcg3": 0.9904041735928579,
|
|
"mean_first_positive_rank": 1.026,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 21.25606323112175,
|
|
"median_score_margin": 19.60146927833557,
|
|
"negative_margin_rate": 0.026,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.999997846186161,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9815,
|
|
"recall1": 0.9815,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99075,
|
|
"ndcg3": 0.9931722004410719,
|
|
"mean_first_positive_rank": 1.0185,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 21.907086151573807,
|
|
"median_score_margin": 19.700664520263672,
|
|
"negative_margin_rate": 0.0185,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999978821873665,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.00013525628542129198,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.979,
|
|
"recall1": 0.979,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9894166666666666,
|
|
"ndcg3": 0.9921840599482149,
|
|
"mean_first_positive_rank": 1.0215,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 19.10362393591553,
|
|
"median_score_margin": 18.17929482460022,
|
|
"negative_margin_rate": 0.021,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999990122318267,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9675,
|
|
"recall1": 0.48375,
|
|
"recall3": 0.99375,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.9875,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9835833333333334,
|
|
"ndcg3": 0.9774938937853556,
|
|
"mean_first_positive_rank": 1.0335,
|
|
"all_evidence_in_top3": 0.9875,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9875,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 18.34438194381073,
|
|
"median_score_margin": 20.793008476495743,
|
|
"negative_margin_rate": 0.0325,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999314472973346,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9925,
|
|
"recall1": 0.9925,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99625,
|
|
"ndcg3": 0.9972319731517859,
|
|
"mean_first_positive_rank": 1.0075,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 18.88056689170003,
|
|
"median_score_margin": 17.6385018825531,
|
|
"negative_margin_rate": 0.0075,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999094858169556,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.979,
|
|
"recall1": 0.979,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9895,
|
|
"ndcg3": 0.9922495248250006,
|
|
"mean_first_positive_rank": 1.021,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 22.230981560852378,
|
|
"median_score_margin": 20.683798789978027,
|
|
"negative_margin_rate": 0.021,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999773879647255,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9725,
|
|
"recall1": 0.9725,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98625,
|
|
"ndcg3": 0.989850568223215,
|
|
"mean_first_positive_rank": 1.0275,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 19.596892535755412,
|
|
"median_score_margin": 18.929328441619873,
|
|
"negative_margin_rate": 0.0275,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.999999755859375,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.5700110253583242,
|
|
"recall1": 0.5700110253583242,
|
|
"recall3": 0.6901874310915105,
|
|
"recall5": 0.7816979051819184,
|
|
"recall8": 0.8820286659316428,
|
|
"evidence_count_in_top3": 0.6901874310915105,
|
|
"hit3": 0.6901874310915105,
|
|
"mrr": 0.6672981046423672,
|
|
"ndcg3": 0.6381109165636338,
|
|
"mean_first_positive_rank": 3.2221609702315326,
|
|
"all_evidence_in_top3": 0.6901874310915105,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 2.0245379494059557,
|
|
"median_score_margin": 1.7993928790092468,
|
|
"negative_margin_rate": 0.42998897464167585,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9998716599294606,
|
|
"unknown_mean_need_prob": 0.043839044658629714,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 9.084831467589538e-07,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"V2-128 v6 best": {
|
|
"checkpoint": "checkpoints\\router_v6_v2_128\\router_best.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": 6000,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 2037774,
|
|
"checkpoint_bytes": 24472973
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 24472973
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 1.2807459999263908,
|
|
"single_query_latency_ms_p50": 1.0074999881908298,
|
|
"single_query_latency_ms_p95": 1.8933000101242214,
|
|
"queries_per_second": 780.7949429921887,
|
|
"samples": 150,
|
|
"batched_qps_64": 62429.27930602223,
|
|
"batched_latency_ms_64": 0.016018125006667105,
|
|
"batched_qps_256": 201612.9031802962,
|
|
"batched_latency_ms_256": 0.004960000001119624
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.4724313324550167,
|
|
"single_query_latency_ms_p50": 0.40469999657943845,
|
|
"single_query_latency_ms_p95": 0.8424999978160486,
|
|
"queries_per_second": 2116.7097338854355,
|
|
"samples": 150,
|
|
"batched_qps_64": 10979.62250423899,
|
|
"batched_latency_ms_64": 0.09107781252168934,
|
|
"batched_qps_256": 12091.087451043528,
|
|
"batched_latency_ms_256": 0.08270554687896947
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.946165936903559,
|
|
"recall1": 0.8919108566296171,
|
|
"recall3": 0.9780509711462895,
|
|
"recall5": 0.984001347254968,
|
|
"recall8": 0.9933198607836533,
|
|
"evidence_count_in_top3": 1.0897608622431796,
|
|
"hit3": 0.9786123273829572,
|
|
"mrr": 0.9648813461092199,
|
|
"ndcg3": 0.9649494672503449,
|
|
"mean_first_positive_rank": 1.1781744695183562,
|
|
"all_evidence_in_top3": 0.9774896149096216,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.99,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 14.959203617273067,
|
|
"median_score_margin": 14.255205273628235,
|
|
"negative_margin_rate": 0.053834063096441,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999601252354449,
|
|
"unknown_mean_need_prob": 0.0012111918483046415,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.010367806648901023,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.976875,
|
|
"recall1": 0.91646875,
|
|
"recall3": 0.999375,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.12375,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9883645833333333,
|
|
"ndcg3": 0.9901670270848972,
|
|
"mean_first_positive_rank": 1.0235625,
|
|
"all_evidence_in_top3": 0.99875,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.99,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 16.402275828796906,
|
|
"median_score_margin": 15.20901894569397,
|
|
"negative_margin_rate": 0.023125,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999701014049351,
|
|
"unknown_mean_need_prob": 6.905754663642937e-05,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.6854130052724078,
|
|
"recall1": 0.6854130052724078,
|
|
"recall3": 0.786174575278266,
|
|
"recall5": 0.836555360281195,
|
|
"recall8": 0.9302870533099004,
|
|
"evidence_count_in_top3": 0.786174575278266,
|
|
"hit3": 0.786174575278266,
|
|
"mrr": 0.7618258791916341,
|
|
"ndcg3": 0.7445377808477717,
|
|
"mean_first_positive_rank": 2.570591681312244,
|
|
"all_evidence_in_top3": 0.786174575278266,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 2.378015720335124,
|
|
"median_score_margin": 3.452073097229004,
|
|
"negative_margin_rate": 0.3145869947275923,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9998671145696143,
|
|
"unknown_mean_need_prob": 0.0960329504062732,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.514018691588785,
|
|
"recall1": 0.514018691588785,
|
|
"recall3": 0.8504672897196262,
|
|
"recall5": 0.9439252336448598,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 0.8504672897196262,
|
|
"hit3": 0.8504672897196262,
|
|
"mrr": 0.6927681352914997,
|
|
"ndcg3": 0.7103867694592852,
|
|
"mean_first_positive_rank": 2.0841121495327104,
|
|
"all_evidence_in_top3": 0.8504672897196262,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.11619492766456069,
|
|
"median_score_margin": 0.07718276977539062,
|
|
"negative_margin_rate": 0.48598130841121495,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 0.9999521859338351,
|
|
"unknown_mean_need_prob": 0.0960329504062732,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9885,
|
|
"recall1": 0.9885,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99425,
|
|
"ndcg3": 0.9957556921660717,
|
|
"mean_first_positive_rank": 1.0115,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 18.511014288797973,
|
|
"median_score_margin": 17.0071280002594,
|
|
"negative_margin_rate": 0.0115,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999475074112415,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9755,
|
|
"recall1": 0.9755,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98775,
|
|
"ndcg3": 0.9909577789625007,
|
|
"mean_first_positive_rank": 1.0245,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 17.432058019019664,
|
|
"median_score_margin": 17.119813442230225,
|
|
"negative_margin_rate": 0.0245,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.999997846186161,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9805,
|
|
"recall1": 0.9805,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99025,
|
|
"ndcg3": 0.9928031301946435,
|
|
"mean_first_positive_rank": 1.0195,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 18.200293550987727,
|
|
"median_score_margin": 18.367231130599976,
|
|
"negative_margin_rate": 0.0195,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999978821873665,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.00013719209501925888,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.98,
|
|
"recall1": 0.98,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9899166666666667,
|
|
"ndcg3": 0.9925531301946434,
|
|
"mean_first_positive_rank": 1.0205,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 17.648428340390325,
|
|
"median_score_margin": 17.865666389465332,
|
|
"negative_margin_rate": 0.02,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999990122318267,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9665,
|
|
"recall1": 0.48325,
|
|
"recall3": 0.995,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.99,
|
|
"hit3": 1.0,
|
|
"mrr": 0.983,
|
|
"ndcg3": 0.9775122185202447,
|
|
"mean_first_positive_rank": 1.035,
|
|
"all_evidence_in_top3": 0.99,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.99,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 10.340590946737677,
|
|
"median_score_margin": 12.604959726333618,
|
|
"negative_margin_rate": 0.0335,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999316669106484,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9855,
|
|
"recall1": 0.9855,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99275,
|
|
"ndcg3": 0.9946484814267861,
|
|
"mean_first_positive_rank": 1.0145,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 16.270763264942914,
|
|
"median_score_margin": 14.272640228271484,
|
|
"negative_margin_rate": 0.0145,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.999909707903862,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.978,
|
|
"recall1": 0.978,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.989,
|
|
"ndcg3": 0.9918804545785721,
|
|
"mean_first_positive_rank": 1.022,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 17.229778190023264,
|
|
"median_score_margin": 17.07593348622322,
|
|
"negative_margin_rate": 0.022,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999774325489997,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9605,
|
|
"recall1": 0.9605,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98,
|
|
"ndcg3": 0.9852253306357154,
|
|
"mean_first_positive_rank": 1.041,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 15.585280029475689,
|
|
"median_score_margin": 15.148456335067749,
|
|
"negative_margin_rate": 0.0395,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.999999755859375,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.675303197353914,
|
|
"recall1": 0.675303197353914,
|
|
"recall3": 0.7899669239250275,
|
|
"recall5": 0.8428886438809261,
|
|
"recall8": 0.934399117971334,
|
|
"evidence_count_in_top3": 0.7899669239250275,
|
|
"hit3": 0.7899669239250275,
|
|
"mrr": 0.7577524621038092,
|
|
"ndcg3": 0.74252336066113,
|
|
"mean_first_positive_rank": 2.541896361631753,
|
|
"all_evidence_in_top3": 0.7899669239250275,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 2.2308930415391117,
|
|
"median_score_margin": 3.026643395423889,
|
|
"negative_margin_rate": 0.324696802646086,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9998721325607783,
|
|
"unknown_mean_need_prob": 0.04431059945842585,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 9.229982535998715e-07,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} |