WpyQwq
643e22ecb9
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线 - 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同） - 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00） - 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00% - 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益） - 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
6132 lines
195 KiB
JSON
6132 lines
195 KiB
JSON
{
|
|
"V2-128 deployed(v3)": {
|
|
"checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_qwen_router_entities\\memory_router_v2.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 2037774,
|
|
"checkpoint_bytes": 8157561
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 8157561
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 1.155849999122438,
|
|
"single_query_latency_ms_p50": 1.155849999122438,
|
|
"single_query_latency_ms_p95": 1.1732999992091209,
|
|
"queries_per_second": 865.1641655571528,
|
|
"batched_qps_64": 54038.53621622001,
|
|
"batched_qps_256": 199554.12120495233,
|
|
"batched_latency_ms_64": 0.01850531250511267,
|
|
"samples": 5,
|
|
"source": "bench_router_latency.py (interleaved round-robin medians)"
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.38441666644454625,
|
|
"single_query_latency_ms_p50": 0.36130000080447644,
|
|
"single_query_latency_ms_p95": 0.5763999979535583,
|
|
"queries_per_second": 2601.3440292507566,
|
|
"samples": 30,
|
|
"batched_qps_64": 12592.97154659022,
|
|
"batched_latency_ms_64": 0.07940937500734435,
|
|
"batched_qps_256": 13508.49346529746,
|
|
"batched_latency_ms_256": 0.07402749999982916
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.4112495789828225,
|
|
"recall1": 0.37027057370607386,
|
|
"recall3": 0.4673009992141013,
|
|
"recall5": 0.4891377568204783,
|
|
"recall8": 0.5160267205568654,
|
|
"evidence_count_in_top3": 0.5420455821264174,
|
|
"hit3": 0.5003368137420007,
|
|
"mrr": 0.476908110965675,
|
|
"ndcg3": 0.44220238482445295,
|
|
"mean_first_positive_rank": 14.38671831144044,
|
|
"all_evidence_in_top3": 0.43426518468620184,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.3715,
|
|
"hop_accuracy": 0.7322992700729927,
|
|
"hop_accuracy_answerable": 0.9010890310991355,
|
|
"hop_under_prediction": 0.046817110138093636,
|
|
"hop_over_prediction": 0.05209385876277085,
|
|
"mean_score_margin": -9.29892828391031,
|
|
"median_score_margin": -9.269989609718323,
|
|
"negative_margin_rate": 0.5887504210171774,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 4042,
|
|
"true_0_pred_2": 64,
|
|
"true_1_pred_0": 834,
|
|
"true_1_pred_1": 14052,
|
|
"true_1_pred_2": 928,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9762653562039442,
|
|
"unknown_mean_need_prob": 0.998855464207621,
|
|
"thr0.30": {
|
|
"need_precision": 0.8126824817518248,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.8966628076710123,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8126824817518248,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.8126824817518248,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.8966628076710123,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8126824817518248,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.8124086257309941,
|
|
"need_recall": 0.998203660042663,
|
|
"need_f1": 0.895773512669387,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8112226277372263,
|
|
"known_question_refusal_rate": 0.001796339957336926,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17782,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 32
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.8115476409032495,
|
|
"need_recall": 0.9925900976759852,
|
|
"need_f1": 0.892985202767537,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8066605839416059,
|
|
"known_question_refusal_rate": 0.00740990232401482,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17682,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 132
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 0.8112270700197692,
|
|
"need_recall": 0.9905130796003143,
|
|
"need_f1": 0.8919499557689878,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8049726277372263,
|
|
"known_question_refusal_rate": 0.00948692039968564,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17645,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 169
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 0.8106525247867189,
|
|
"need_recall": 0.9868081284383069,
|
|
"need_f1": 0.8900984835059115,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8019616788321168,
|
|
"known_question_refusal_rate": 0.01319187156169305,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17579,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 235
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_2": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.9999341862276196,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 64,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.442375,
|
|
"recall1": 0.39675,
|
|
"recall3": 0.49153125,
|
|
"recall5": 0.50303125,
|
|
"recall8": 0.51234375,
|
|
"evidence_count_in_top3": 0.57475,
|
|
"hit3": 0.5283125,
|
|
"mrr": 0.500402578720693,
|
|
"ndcg3": 0.4692792057097287,
|
|
"mean_first_positive_rank": 14.9525,
|
|
"all_evidence_in_top3": 0.45475,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.3715,
|
|
"hop_accuracy": 0.74955,
|
|
"hop_accuracy_answerable": 0.9369375,
|
|
"hop_under_prediction": 0.0160625,
|
|
"hop_over_prediction": 0.047,
|
|
"mean_score_margin": -8.091850390457548,
|
|
"median_score_margin": -6.662378132343292,
|
|
"negative_margin_rate": 0.557625,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 4000,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 257,
|
|
"true_1_pred_1": 12991,
|
|
"true_1_pred_2": 752,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9821553228273988,
|
|
"unknown_mean_need_prob": 0.9988265322595835,
|
|
"thr0.50": {
|
|
"need_precision": 0.8,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.888888888888889,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 0,
|
|
"fp": 4000,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.14528412419449327,
|
|
"recall1": 0.14528412419449327,
|
|
"recall3": 0.2677211482132396,
|
|
"recall5": 0.38488576449912126,
|
|
"recall8": 0.5653192735793791,
|
|
"evidence_count_in_top3": 0.2677211482132396,
|
|
"hit3": 0.2677211482132396,
|
|
"mrr": 0.2801716446629072,
|
|
"ndcg3": 0.21516992510461316,
|
|
"mean_first_positive_rank": 9.195664909197422,
|
|
"all_evidence_in_top3": 0.2677211482132396,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.5740740740740741,
|
|
"hop_accuracy_answerable": 0.5811364967779731,
|
|
"hop_under_prediction": 0.3380199179847686,
|
|
"hop_over_prediction": 0.08084358523725835,
|
|
"mean_score_margin": -19.359866383266713,
|
|
"median_score_margin": -18.159542322158813,
|
|
"negative_margin_rate": 0.8547158758055068,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 21,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 577,
|
|
"true_1_pred_1": 992,
|
|
"true_1_pred_2": 138
|
|
},
|
|
"known_mean_need_prob": 0.9195722371566764,
|
|
"unknown_mean_need_prob": 0.9999671209426153,
|
|
"thr0.50": {
|
|
"need_precision": 0.9876179245283019,
|
|
"need_recall": 0.981253661394259,
|
|
"need_f1": 0.9844255069056715,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.9693287037037037,
|
|
"known_question_refusal_rate": 0.018746338605741066,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 1675,
|
|
"tn": 0,
|
|
"fp": 21,
|
|
"fn": 32
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.028037383177570093,
|
|
"recall5": 0.07476635514018691,
|
|
"recall8": 0.2803738317757009,
|
|
"evidence_count_in_top3": 0.028037383177570093,
|
|
"hit3": 0.028037383177570093,
|
|
"mrr": 0.1023068389893556,
|
|
"ndcg3": 0.01524233414552764,
|
|
"mean_first_positive_rank": 12.598130841121495,
|
|
"all_evidence_in_top3": 0.028037383177570093,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.5390625,
|
|
"hop_accuracy_answerable": 0.6448598130841121,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.35514018691588783,
|
|
"mean_score_margin": -29.291684916086286,
|
|
"median_score_margin": -29.07153034210205,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 21,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 69,
|
|
"true_1_pred_2": 38
|
|
},
|
|
"known_mean_need_prob": 0.9999633771236812,
|
|
"unknown_mean_need_prob": 0.9999671209426153,
|
|
"thr0.50": {
|
|
"need_precision": 0.8359375,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9106382978723404,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8359375,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 0,
|
|
"fp": 21,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.796,
|
|
"recall1": 0.796,
|
|
"recall3": 0.9125,
|
|
"recall5": 0.919,
|
|
"recall8": 0.927,
|
|
"evidence_count_in_top3": 0.9125,
|
|
"hit3": 0.9125,
|
|
"mrr": 0.8582544951434297,
|
|
"ndcg3": 0.8678012294946459,
|
|
"mean_first_positive_rank": 2.8275,
|
|
"all_evidence_in_top3": 0.9125,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.4955,
|
|
"hop_accuracy_answerable": 0.4955,
|
|
"hop_under_prediction": 0.1285,
|
|
"hop_over_prediction": 0.376,
|
|
"mean_score_margin": 21.036133953779935,
|
|
"median_score_margin": 21.400100469589233,
|
|
"negative_margin_rate": 0.204,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 257,
|
|
"true_1_pred_1": 991,
|
|
"true_1_pred_2": 752
|
|
},
|
|
"known_mean_need_prob": 0.9764632563889026,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.231,
|
|
"recall1": 0.231,
|
|
"recall3": 0.2535,
|
|
"recall5": 0.255,
|
|
"recall8": 0.257,
|
|
"evidence_count_in_top3": 0.2535,
|
|
"hit3": 0.2535,
|
|
"mrr": 0.26603145346142226,
|
|
"ndcg3": 0.24486859507142916,
|
|
"mean_first_positive_rank": 23.789,
|
|
"all_evidence_in_top3": 0.2535,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -27.718347060799598,
|
|
"median_score_margin": -38.59446668624878,
|
|
"negative_margin_rate": 0.769,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9723067943453789,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.2255,
|
|
"recall1": 0.2255,
|
|
"recall3": 0.2455,
|
|
"recall5": 0.2475,
|
|
"recall8": 0.2485,
|
|
"evidence_count_in_top3": 0.2455,
|
|
"hit3": 0.2455,
|
|
"mrr": 0.2596338049221357,
|
|
"ndcg3": 0.23792220044107196,
|
|
"mean_first_positive_rank": 24.0695,
|
|
"all_evidence_in_top3": 0.2455,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -28.903205164782705,
|
|
"median_score_margin": -39.01692509651184,
|
|
"negative_margin_rate": 0.7745,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9724746745526791,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.99774328109622,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 2000,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5255,
|
|
"recall1": 0.5255,
|
|
"recall3": 0.6245,
|
|
"recall5": 0.6265,
|
|
"recall8": 0.6295,
|
|
"evidence_count_in_top3": 0.6245,
|
|
"hit3": 0.6245,
|
|
"mrr": 0.5855216777342396,
|
|
"ndcg3": 0.585867169546431,
|
|
"mean_first_positive_rank": 12.2085,
|
|
"all_evidence_in_top3": 0.6245,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -1.0039909677058458,
|
|
"median_score_margin": 2.929476022720337,
|
|
"negative_margin_rate": 0.4745,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9875293919146061,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.73,
|
|
"recall1": 0.365,
|
|
"recall3": 0.66575,
|
|
"recall5": 0.72725,
|
|
"recall8": 0.77375,
|
|
"evidence_count_in_top3": 1.3315,
|
|
"hit3": 0.96,
|
|
"mrr": 0.84532448592014,
|
|
"ndcg3": 0.6658792959224611,
|
|
"mean_first_positive_rank": 1.59,
|
|
"all_evidence_in_top3": 0.3715,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.3715,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 2.786984693795443,
|
|
"median_score_margin": 5.27826452255249,
|
|
"negative_margin_rate": 0.27,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9993210740983486,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.443,
|
|
"recall1": 0.443,
|
|
"recall3": 0.5525,
|
|
"recall5": 0.5615,
|
|
"recall8": 0.5665,
|
|
"evidence_count_in_top3": 0.5525,
|
|
"hit3": 0.5525,
|
|
"mrr": 0.5118924660255056,
|
|
"ndcg3": 0.5092063534375025,
|
|
"mean_first_positive_rank": 13.8255,
|
|
"all_evidence_in_top3": 0.5525,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -2.995655853047967,
|
|
"median_score_margin": -5.5251545906066895,
|
|
"negative_margin_rate": 0.557,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9880377069413662,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.249,
|
|
"recall1": 0.249,
|
|
"recall3": 0.287,
|
|
"recall5": 0.293,
|
|
"recall8": 0.3005,
|
|
"evidence_count_in_top3": 0.287,
|
|
"hit3": 0.287,
|
|
"mrr": 0.2919631936667751,
|
|
"ndcg3": 0.27173149797678653,
|
|
"mean_first_positive_rank": 22.067,
|
|
"all_evidence_in_top3": 0.287,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -19.72415616109967,
|
|
"median_score_margin": -21.928287506103516,
|
|
"negative_margin_rate": 0.751,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9656237106621265,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.339,
|
|
"recall1": 0.339,
|
|
"recall3": 0.391,
|
|
"recall5": 0.3945,
|
|
"recall8": 0.396,
|
|
"evidence_count_in_top3": 0.391,
|
|
"hit3": 0.391,
|
|
"mrr": 0.3845990528918967,
|
|
"ndcg3": 0.3709573037875013,
|
|
"mean_first_positive_rank": 19.243,
|
|
"all_evidence_in_top3": 0.391,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -8.212566563799978,
|
|
"median_score_margin": -16.937832549214363,
|
|
"negative_margin_rate": 0.661,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9954859737157822,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.13671444321940462,
|
|
"recall1": 0.13671444321940462,
|
|
"recall3": 0.2535832414553473,
|
|
"recall5": 0.36659316427783906,
|
|
"recall8": 0.5485115766262404,
|
|
"evidence_count_in_top3": 0.2535832414553473,
|
|
"hit3": 0.2535832414553473,
|
|
"mrr": 0.26968017045834825,
|
|
"ndcg3": 0.2033770627933551,
|
|
"mean_first_positive_rank": 9.396361631753033,
|
|
"all_evidence_in_top3": 0.2535832414553473,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.5526041666666667,
|
|
"hop_accuracy_answerable": 0.5848952590959207,
|
|
"hop_under_prediction": 0.31808158765159866,
|
|
"hop_over_prediction": 0.0970231532524807,
|
|
"mean_score_margin": -19.945701324287494,
|
|
"median_score_margin": -18.862584590911865,
|
|
"negative_margin_rate": 0.8632855567805954,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 42,
|
|
"true_0_pred_2": 64,
|
|
"true_1_pred_0": 577,
|
|
"true_1_pred_1": 1061,
|
|
"true_1_pred_2": 176
|
|
},
|
|
"known_mean_need_prob": 0.9243141621712682,
|
|
"unknown_mean_need_prob": 0.9999472358316746,
|
|
"thr0.50": {
|
|
"need_precision": 0.9438559322033898,
|
|
"need_recall": 0.9823594266813671,
|
|
"need_f1": 0.9627228525121555,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.928125,
|
|
"known_question_refusal_rate": 0.017640573318632856,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 1782,
|
|
"tn": 0,
|
|
"fp": 106,
|
|
"fn": 32
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.9999097834229469,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 2000,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"V2-128 v6 best": {
|
|
"checkpoint": "checkpoints\\router_v6_v2_128\\router_best.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": 6000,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 2037774,
|
|
"checkpoint_bytes": 24472973
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 24472973
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 1.167899999927613,
|
|
"single_query_latency_ms_p50": 1.167899999927613,
|
|
"single_query_latency_ms_p95": 1.196150000396301,
|
|
"queries_per_second": 856.2376916362535,
|
|
"batched_qps_64": 54149.18098951274,
|
|
"batched_qps_256": 196829.20449441153,
|
|
"batched_latency_ms_64": 0.018467500001406734,
|
|
"samples": 5,
|
|
"source": "bench_router_latency.py (interleaved round-robin medians)"
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.33882666709056747,
|
|
"single_query_latency_ms_p50": 0.338400001055561,
|
|
"single_query_latency_ms_p95": 0.4341000021668151,
|
|
"queries_per_second": 2951.3615577746205,
|
|
"samples": 30,
|
|
"batched_qps_64": 12676.831899779869,
|
|
"batched_latency_ms_64": 0.07888406250913249,
|
|
"batched_qps_256": 13789.017263119036,
|
|
"batched_latency_ms_256": 0.07252148437544292
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.946165936903559,
|
|
"recall1": 0.8919108566296171,
|
|
"recall3": 0.9780509711462895,
|
|
"recall5": 0.984001347254968,
|
|
"recall8": 0.9933198607836533,
|
|
"evidence_count_in_top3": 1.0897608622431796,
|
|
"hit3": 0.9786123273829572,
|
|
"mrr": 0.9648813461092199,
|
|
"ndcg3": 0.9649494672503449,
|
|
"mean_first_positive_rank": 1.1781744695183562,
|
|
"all_evidence_in_top3": 0.9774896149096216,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.99,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 14.959203617273067,
|
|
"median_score_margin": 14.255205273628235,
|
|
"negative_margin_rate": 0.053834063096441,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999601252354449,
|
|
"unknown_mean_need_prob": 0.0012111918483046415,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.010367806648901023,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.976875,
|
|
"recall1": 0.91646875,
|
|
"recall3": 0.999375,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.12375,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9883645833333333,
|
|
"ndcg3": 0.9901670270848972,
|
|
"mean_first_positive_rank": 1.0235625,
|
|
"all_evidence_in_top3": 0.99875,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.99,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 16.402275828796906,
|
|
"median_score_margin": 15.20901894569397,
|
|
"negative_margin_rate": 0.023125,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999701014049351,
|
|
"unknown_mean_need_prob": 6.905754663642937e-05,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.6854130052724078,
|
|
"recall1": 0.6854130052724078,
|
|
"recall3": 0.786174575278266,
|
|
"recall5": 0.836555360281195,
|
|
"recall8": 0.9302870533099004,
|
|
"evidence_count_in_top3": 0.786174575278266,
|
|
"hit3": 0.786174575278266,
|
|
"mrr": 0.7618258791916341,
|
|
"ndcg3": 0.7445377808477717,
|
|
"mean_first_positive_rank": 2.570591681312244,
|
|
"all_evidence_in_top3": 0.786174575278266,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 2.378015720335124,
|
|
"median_score_margin": 3.452073097229004,
|
|
"negative_margin_rate": 0.3145869947275923,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9998671145696143,
|
|
"unknown_mean_need_prob": 0.0960329504062732,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.514018691588785,
|
|
"recall1": 0.514018691588785,
|
|
"recall3": 0.8504672897196262,
|
|
"recall5": 0.9439252336448598,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 0.8504672897196262,
|
|
"hit3": 0.8504672897196262,
|
|
"mrr": 0.6927681352914997,
|
|
"ndcg3": 0.7103867694592852,
|
|
"mean_first_positive_rank": 2.0841121495327104,
|
|
"all_evidence_in_top3": 0.8504672897196262,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.11619492766456069,
|
|
"median_score_margin": 0.07718276977539062,
|
|
"negative_margin_rate": 0.48598130841121495,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 0.9999521859338351,
|
|
"unknown_mean_need_prob": 0.0960329504062732,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9885,
|
|
"recall1": 0.9885,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99425,
|
|
"ndcg3": 0.9957556921660717,
|
|
"mean_first_positive_rank": 1.0115,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 18.511014288797973,
|
|
"median_score_margin": 17.0071280002594,
|
|
"negative_margin_rate": 0.0115,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999475074112415,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9755,
|
|
"recall1": 0.9755,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98775,
|
|
"ndcg3": 0.9909577789625007,
|
|
"mean_first_positive_rank": 1.0245,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 17.432058019019664,
|
|
"median_score_margin": 17.119813442230225,
|
|
"negative_margin_rate": 0.0245,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.999997846186161,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9805,
|
|
"recall1": 0.9805,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99025,
|
|
"ndcg3": 0.9928031301946435,
|
|
"mean_first_positive_rank": 1.0195,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 18.200293550987727,
|
|
"median_score_margin": 18.367231130599976,
|
|
"negative_margin_rate": 0.0195,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999978821873665,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.00013719209501925888,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.98,
|
|
"recall1": 0.98,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9899166666666667,
|
|
"ndcg3": 0.9925531301946434,
|
|
"mean_first_positive_rank": 1.0205,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 17.648428340390325,
|
|
"median_score_margin": 17.865666389465332,
|
|
"negative_margin_rate": 0.02,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999990122318267,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9665,
|
|
"recall1": 0.48325,
|
|
"recall3": 0.995,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.99,
|
|
"hit3": 1.0,
|
|
"mrr": 0.983,
|
|
"ndcg3": 0.9775122185202447,
|
|
"mean_first_positive_rank": 1.035,
|
|
"all_evidence_in_top3": 0.99,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.99,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 10.340590946737677,
|
|
"median_score_margin": 12.604959726333618,
|
|
"negative_margin_rate": 0.0335,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999316669106484,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9855,
|
|
"recall1": 0.9855,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99275,
|
|
"ndcg3": 0.9946484814267861,
|
|
"mean_first_positive_rank": 1.0145,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 16.270763264942914,
|
|
"median_score_margin": 14.272640228271484,
|
|
"negative_margin_rate": 0.0145,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.999909707903862,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.978,
|
|
"recall1": 0.978,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.989,
|
|
"ndcg3": 0.9918804545785721,
|
|
"mean_first_positive_rank": 1.022,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 17.229778190023264,
|
|
"median_score_margin": 17.07593348622322,
|
|
"negative_margin_rate": 0.022,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999774325489997,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9605,
|
|
"recall1": 0.9605,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98,
|
|
"ndcg3": 0.9852253306357154,
|
|
"mean_first_positive_rank": 1.041,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 15.585280029475689,
|
|
"median_score_margin": 15.148456335067749,
|
|
"negative_margin_rate": 0.0395,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.999999755859375,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.675303197353914,
|
|
"recall1": 0.675303197353914,
|
|
"recall3": 0.7899669239250275,
|
|
"recall5": 0.8428886438809261,
|
|
"recall8": 0.934399117971334,
|
|
"evidence_count_in_top3": 0.7899669239250275,
|
|
"hit3": 0.7899669239250275,
|
|
"mrr": 0.7577524621038092,
|
|
"ndcg3": 0.74252336066113,
|
|
"mean_first_positive_rank": 2.541896361631753,
|
|
"all_evidence_in_top3": 0.7899669239250275,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 2.2308930415391117,
|
|
"median_score_margin": 3.026643395423889,
|
|
"negative_margin_rate": 0.324696802646086,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9998721325607783,
|
|
"unknown_mean_need_prob": 0.04431059945842585,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 9.229982535998715e-07,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"V2-128 v6 final": {
|
|
"checkpoint": "checkpoints\\router_v6_v2_128\\memory_router_v2.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 2037774,
|
|
"checkpoint_bytes": 8156921
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 8156921
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 1.165349998700549,
|
|
"single_query_latency_ms_p50": 1.165349998700549,
|
|
"single_query_latency_ms_p95": 1.1786999984906288,
|
|
"queries_per_second": 858.1112979920828,
|
|
"batched_qps_64": 53471.46796235134,
|
|
"batched_qps_256": 198025.9290094514,
|
|
"batched_latency_ms_64": 0.018701562498790736,
|
|
"samples": 5,
|
|
"source": "bench_router_latency.py (interleaved round-robin medians)"
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.3544133333586312,
|
|
"single_query_latency_ms_p50": 0.3417000007175375,
|
|
"single_query_latency_ms_p95": 0.4805999997188337,
|
|
"queries_per_second": 2821.564275032788,
|
|
"samples": 30,
|
|
"batched_qps_64": 13137.043999093688,
|
|
"batched_latency_ms_64": 0.07612062501038963,
|
|
"batched_qps_256": 13174.159298667208,
|
|
"batched_latency_ms_256": 0.07590617187247517
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.9436959694622207,
|
|
"recall1": 0.8882059054676097,
|
|
"recall3": 0.9639609296059279,
|
|
"recall5": 0.973981138430448,
|
|
"recall8": 0.9864151790726395,
|
|
"evidence_count_in_top3": 1.0717974626698104,
|
|
"hit3": 0.9683956438756035,
|
|
"mrr": 0.9597054333626427,
|
|
"ndcg3": 0.9530820526370456,
|
|
"mean_first_positive_rank": 1.2573818345121814,
|
|
"all_evidence_in_top3": 0.9595262153362524,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.921,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 51.315328869764656,
|
|
"median_score_margin": 56.234654903411865,
|
|
"negative_margin_rate": 0.05630403053777928,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999984274064,
|
|
"unknown_mean_need_prob": 4.836221294362452e-06,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 4.8406160974828055e-06,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.9838125,
|
|
"recall1": 0.92203125,
|
|
"recall3": 0.9950625,
|
|
"recall5": 0.99978125,
|
|
"recall8": 0.999875,
|
|
"evidence_count_in_top3": 1.115125,
|
|
"hit3": 1.0,
|
|
"mrr": 0.991875,
|
|
"ndcg3": 0.9876817889701721,
|
|
"mean_first_positive_rank": 1.016375,
|
|
"all_evidence_in_top3": 0.990125,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.921,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 57.110110210259094,
|
|
"median_score_margin": 59.90444374084473,
|
|
"negative_margin_rate": 0.0161875,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999999701977,
|
|
"unknown_mean_need_prob": 6.379481702447893e-11,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.5940246045694201,
|
|
"recall1": 0.5940246045694201,
|
|
"recall3": 0.6842413591095489,
|
|
"recall5": 0.7352079671939075,
|
|
"recall8": 0.8611599297012302,
|
|
"evidence_count_in_top3": 0.6842413591095489,
|
|
"hit3": 0.6842413591095489,
|
|
"mrr": 0.6760273585431921,
|
|
"ndcg3": 0.6466497456649108,
|
|
"mean_first_positive_rank": 3.4463971880492092,
|
|
"all_evidence_in_top3": 0.6842413591095489,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.27154229221702947,
|
|
"median_score_margin": 7.975786209106445,
|
|
"negative_margin_rate": 0.40597539543058,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9999999838679872,
|
|
"unknown_mean_need_prob": 0.0004654159529796483,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.5233644859813084,
|
|
"recall1": 0.5233644859813084,
|
|
"recall3": 0.7757009345794392,
|
|
"recall5": 0.9252336448598131,
|
|
"recall8": 0.9719626168224299,
|
|
"evidence_count_in_top3": 0.7757009345794392,
|
|
"hit3": 0.7757009345794392,
|
|
"mrr": 0.674896157840083,
|
|
"ndcg3": 0.6678873486315127,
|
|
"mean_first_positive_rank": 2.3738317757009346,
|
|
"all_evidence_in_top3": 0.7757009345794392,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.8786688875929217,
|
|
"median_score_margin": 0.4756507873535156,
|
|
"negative_margin_rate": 0.4766355140186916,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0004654159529796483,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.993,
|
|
"recall1": 0.993,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9965,
|
|
"ndcg3": 0.9974165082750002,
|
|
"mean_first_positive_rank": 1.007,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 54.4210530708991,
|
|
"median_score_margin": 56.92102670669556,
|
|
"negative_margin_rate": 0.007,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.975,
|
|
"recall1": 0.975,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9875,
|
|
"ndcg3": 0.9907732438392864,
|
|
"mean_first_positive_rank": 1.025,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 64.29146421498805,
|
|
"median_score_margin": 73.40720510482788,
|
|
"negative_margin_rate": 0.025,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9845,
|
|
"recall1": 0.9845,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99225,
|
|
"ndcg3": 0.9942794111803576,
|
|
"mean_first_positive_rank": 1.0155,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 65.52974343589041,
|
|
"median_score_margin": 73.97303533554077,
|
|
"negative_margin_rate": 0.0155,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 1.275862421441728e-10,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9795,
|
|
"recall1": 0.9795,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9896666666666666,
|
|
"ndcg3": 0.9923685950714292,
|
|
"mean_first_positive_rank": 1.021,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 60.150181606799364,
|
|
"median_score_margin": 68.01980018615723,
|
|
"negative_margin_rate": 0.0205,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9885,
|
|
"recall1": 0.49425,
|
|
"recall3": 0.9605,
|
|
"recall5": 0.99825,
|
|
"recall8": 0.999,
|
|
"evidence_count_in_top3": 1.921,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99425,
|
|
"ndcg3": 0.9452009954703022,
|
|
"mean_first_positive_rank": 1.0115,
|
|
"all_evidence_in_top3": 0.921,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.921,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 55.6439196386747,
|
|
"median_score_margin": 62.50904202461243,
|
|
"negative_margin_rate": 0.0115,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999997615814,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.995,
|
|
"recall1": 0.995,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9975,
|
|
"ndcg3": 0.9981546487678573,
|
|
"mean_first_positive_rank": 1.005,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 48.13904330630228,
|
|
"median_score_margin": 49.25835180282593,
|
|
"negative_margin_rate": 0.005,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9805,
|
|
"recall1": 0.9805,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9901666666666666,
|
|
"ndcg3": 0.9927376653178577,
|
|
"mean_first_positive_rank": 1.02,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 50.85701001610607,
|
|
"median_score_margin": 54.9089093208313,
|
|
"negative_margin_rate": 0.0195,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9745,
|
|
"recall1": 0.9745,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9871666666666666,
|
|
"ndcg3": 0.9905232438392865,
|
|
"mean_first_positive_rank": 1.026,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 57.84846639241278,
|
|
"median_score_margin": 63.093711376190186,
|
|
"negative_margin_rate": 0.0255,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.5898566703417861,
|
|
"recall1": 0.5898566703417861,
|
|
"recall3": 0.6896361631753032,
|
|
"recall5": 0.7464167585446527,
|
|
"recall8": 0.8676957001102535,
|
|
"evidence_count_in_top3": 0.6896361631753032,
|
|
"hit3": 0.6896361631753032,
|
|
"mrr": 0.6759606339151697,
|
|
"ndcg3": 0.6479024598421029,
|
|
"mean_first_positive_rank": 3.383131201764057,
|
|
"all_evidence_in_top3": 0.6896361631753032,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.20369631854576997,
|
|
"median_score_margin": 6.827788352966309,
|
|
"negative_margin_rate": 0.4101433296582139,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9999999848195447,
|
|
"unknown_mean_need_prob": 0.00018733273071117103,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 3.3919047850488803e-15,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"XL-128 v6 best": {
|
|
"checkpoint": "checkpoints\\router_v6_xl128\\router_best.pt",
|
|
"router": {
|
|
"kind": "router_xl",
|
|
"step": 1000,
|
|
"arch_config": {
|
|
"arch": "router_xl",
|
|
"arch_version": 1,
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3,
|
|
"encoder_layers": 2,
|
|
"encoder_hidden": 512,
|
|
"pair_blocks": 1,
|
|
"pair_hidden": 512,
|
|
"pair_expansion": 2,
|
|
"pair_dropout": 0.05,
|
|
"use_interaction": true,
|
|
"policy_layers": 2,
|
|
"policy_hidden": 512,
|
|
"policy_dropout": 0.0,
|
|
"learnable_cosine_scale": true
|
|
},
|
|
"parameters": 6714639,
|
|
"checkpoint_bytes": 81666921
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 81666921
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 1.578349998453632,
|
|
"single_query_latency_ms_p50": 1.578349998453632,
|
|
"single_query_latency_ms_p95": 1.6234499998972751,
|
|
"queries_per_second": 633.5730357523598,
|
|
"batched_qps_64": 38630.547104467114,
|
|
"batched_qps_256": 141500.569342867,
|
|
"batched_latency_ms_64": 0.02588625000043976,
|
|
"samples": 5,
|
|
"source": "bench_router_latency.py (interleaved round-robin medians)"
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 1.4055266667128308,
|
|
"single_query_latency_ms_p50": 1.3804000009258743,
|
|
"single_query_latency_ms_p95": 1.7585999994480517,
|
|
"queries_per_second": 711.4770738137224,
|
|
"samples": 30,
|
|
"batched_qps_64": 2431.7772491816463,
|
|
"batched_latency_ms_64": 0.4112218750037755,
|
|
"batched_qps_256": 2460.1366567422874,
|
|
"batched_latency_ms_256": 0.4064814843758313
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.9566071629055799,
|
|
"recall1": 0.9027730998091389,
|
|
"recall3": 0.9874817559223082,
|
|
"recall5": 0.9922532839339845,
|
|
"recall8": 0.9991018300213316,
|
|
"evidence_count_in_top3": 1.0992477826428653,
|
|
"hit3": 0.9879869765353093,
|
|
"mrr": 0.9737093504776227,
|
|
"ndcg3": 0.975110976145454,
|
|
"mean_first_positive_rank": 1.098574155158864,
|
|
"all_evidence_in_top3": 0.9869765353093073,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.991,
|
|
"hop_accuracy": 0.9995437956204379,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 16.798009918724365,
|
|
"median_score_margin": 17.34114497900009,
|
|
"negative_margin_rate": 0.04339283709442012,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4096,
|
|
"true_0_pred_1": 10,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9900794763436311,
|
|
"unknown_mean_need_prob": 0.036242366845311694,
|
|
"thr0.30": {
|
|
"need_precision": 0.9964201812283253,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9982068810938025,
|
|
"specificity_unknown_refusal": 0.9844130540672187,
|
|
"abstention_accuracy": 0.997080291970803,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.015586945932781296,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4042,
|
|
"fp": 64,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.998150949739452,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9990746193320434,
|
|
"specificity_unknown_refusal": 0.9919629810034096,
|
|
"abstention_accuracy": 0.9984945255474452,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.008037018996590355,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4073,
|
|
"fp": 33,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.9990466042285907,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9995230747650443,
|
|
"specificity_unknown_refusal": 0.995859717486605,
|
|
"abstention_accuracy": 0.9992244525547446,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.004140282513395032,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4089,
|
|
"fp": 17,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.999326825984517,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9996632996632996,
|
|
"specificity_unknown_refusal": 0.9970774476376035,
|
|
"abstention_accuracy": 0.9994525547445255,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0029225523623964927,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4094,
|
|
"fp": 12,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 0.9996632996632997,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9998316214850985,
|
|
"specificity_unknown_refusal": 0.9985387238188017,
|
|
"abstention_accuracy": 0.9997262773722628,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0014612761811982464,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4100,
|
|
"fp": 6,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.3642658165190369,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 0.953125,
|
|
"abstention_accuracy": 0.953125,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.046875,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 61,
|
|
"fp": 3,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.9765625,
|
|
"recall1": 0.916625,
|
|
"recall3": 0.9994375,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.123875,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9882291666666666,
|
|
"ndcg3": 0.9904076605853642,
|
|
"mean_first_positive_rank": 1.02375,
|
|
"all_evidence_in_top3": 0.998875,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.991,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 17.91327105785167,
|
|
"median_score_margin": 18.24349021911621,
|
|
"negative_margin_rate": 0.0234375,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9914327386058867,
|
|
"unknown_mean_need_prob": 0.027471872764872386,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.8060925600468658,
|
|
"recall1": 0.8060925600468658,
|
|
"recall3": 0.8840070298769771,
|
|
"recall5": 0.9197422378441711,
|
|
"recall8": 0.9906268306971294,
|
|
"evidence_count_in_top3": 0.8840070298769771,
|
|
"hit3": 0.8840070298769771,
|
|
"mrr": 0.8601396423013294,
|
|
"ndcg3": 0.8521063487572197,
|
|
"mean_first_positive_rank": 1.7311072056239016,
|
|
"all_evidence_in_top3": 0.8840070298769771,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.9971064814814815,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 7.396975287374308,
|
|
"median_score_margin": 7.794462203979492,
|
|
"negative_margin_rate": 0.19390743995313414,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 16,
|
|
"true_0_pred_1": 5,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.98037854359034,
|
|
"unknown_mean_need_prob": 0.37168226071766447,
|
|
"thr0.50": {
|
|
"need_precision": 0.9959159859976663,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.997953814674072,
|
|
"specificity_unknown_refusal": 0.6666666666666666,
|
|
"abstention_accuracy": 0.9959490740740741,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.3333333333333333,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 14,
|
|
"fp": 7,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.37383177570093457,
|
|
"recall1": 0.37383177570093457,
|
|
"recall3": 0.8504672897196262,
|
|
"recall5": 0.9906542056074766,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 0.8504672897196262,
|
|
"hit3": 0.8504672897196262,
|
|
"mrr": 0.6143302180685358,
|
|
"ndcg3": 0.6500824519693008,
|
|
"mean_first_positive_rank": 2.196261682242991,
|
|
"all_evidence_in_top3": 0.8504672897196262,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.9609375,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.007242532534019969,
|
|
"median_score_margin": -0.2684917449951172,
|
|
"negative_margin_rate": 0.6261682242990654,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 16,
|
|
"true_0_pred_1": 5,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 0.9424841119864277,
|
|
"unknown_mean_need_prob": 0.37168226071766447,
|
|
"thr0.50": {
|
|
"need_precision": 0.9385964912280702,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9683257918552036,
|
|
"specificity_unknown_refusal": 0.6666666666666666,
|
|
"abstention_accuracy": 0.9453125,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.3333333333333333,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 14,
|
|
"fp": 7,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9785,
|
|
"recall1": 0.9785,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98925,
|
|
"ndcg3": 0.9920649897017864,
|
|
"mean_first_positive_rank": 1.0215,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 19.512430111046882,
|
|
"median_score_margin": 19.13399839401245,
|
|
"negative_margin_rate": 0.0215,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9832325124442577,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.978,
|
|
"recall1": 0.978,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.989,
|
|
"ndcg3": 0.9918804545785721,
|
|
"mean_first_positive_rank": 1.022,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 18.416588767559734,
|
|
"median_score_margin": 18.676148295402527,
|
|
"negative_margin_rate": 0.022,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9980810531675816,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.985,
|
|
"recall1": 0.985,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9924166666666666,
|
|
"ndcg3": 0.9943984814267861,
|
|
"mean_first_positive_rank": 1.0155,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 18.82138747824775,
|
|
"median_score_margin": 18.756187587976456,
|
|
"negative_margin_rate": 0.015,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9980952816009522,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.05116949350852519,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9765,
|
|
"recall1": 0.9765,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98825,
|
|
"ndcg3": 0.9913268492089293,
|
|
"mean_first_positive_rank": 1.0235,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 15.71517678581737,
|
|
"median_score_margin": 16.085471630096436,
|
|
"negative_margin_rate": 0.0235,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9980619028508663,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.959,
|
|
"recall1": 0.4795,
|
|
"recall3": 0.9955,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.991,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9793333333333334,
|
|
"ndcg3": 0.9775264704150523,
|
|
"mean_first_positive_rank": 1.042,
|
|
"all_evidence_in_top3": 0.991,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.991,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 17.043865082176403,
|
|
"median_score_margin": 20.163950562477112,
|
|
"negative_margin_rate": 0.041,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9835184065401554,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.987,
|
|
"recall1": 0.987,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9935,
|
|
"ndcg3": 0.995202086796429,
|
|
"mean_first_positive_rank": 1.013,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 18.87393392176647,
|
|
"median_score_margin": 18.269309908151627,
|
|
"negative_margin_rate": 0.013,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9775506886839866,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.977,
|
|
"recall1": 0.977,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9885,
|
|
"ndcg3": 0.9915113843321435,
|
|
"mean_first_positive_rank": 1.023,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 17.55705898122629,
|
|
"median_score_margin": 17.530375365167856,
|
|
"negative_margin_rate": 0.023,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9958070812821388,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9715,
|
|
"recall1": 0.9715,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9855833333333334,
|
|
"ndcg3": 0.9893505682232151,
|
|
"mean_first_positive_rank": 1.0295,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 17.365727334972473,
|
|
"median_score_margin": 16.746063470840454,
|
|
"negative_margin_rate": 0.0285,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9971149822771549,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.7805953693495039,
|
|
"recall1": 0.7805953693495039,
|
|
"recall3": 0.8820286659316428,
|
|
"recall5": 0.9239250275633958,
|
|
"recall8": 0.9911797133406836,
|
|
"evidence_count_in_top3": 0.8820286659316428,
|
|
"hit3": 0.8820286659316428,
|
|
"mrr": 0.8456404094496708,
|
|
"ndcg3": 0.8401898344483404,
|
|
"mean_first_positive_rank": 1.7585446527012127,
|
|
"all_evidence_in_top3": 0.8820286659316428,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.9947916666666666,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 6.961086971625735,
|
|
"median_score_margin": 7.221376895904541,
|
|
"negative_margin_rate": 0.21940463065049615,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 96,
|
|
"true_0_pred_1": 10,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9781433152653022,
|
|
"unknown_mean_need_prob": 0.3672044076166063,
|
|
"thr0.50": {
|
|
"need_precision": 0.9907154560349536,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9953360768175583,
|
|
"specificity_unknown_refusal": 0.839622641509434,
|
|
"abstention_accuracy": 0.9911458333333333,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.16037735849056603,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 89,
|
|
"fp": 17,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.0037742520212195813,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"XL-128 v6 final": {
|
|
"checkpoint": "checkpoints\\router_v6_xl128\\memory_router_xl.pt",
|
|
"router": {
|
|
"kind": "router_xl",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_xl",
|
|
"arch_version": 1,
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3,
|
|
"encoder_layers": 2,
|
|
"encoder_hidden": 512,
|
|
"pair_blocks": 1,
|
|
"pair_hidden": 512,
|
|
"pair_expansion": 2,
|
|
"pair_dropout": 0.05,
|
|
"use_interaction": true,
|
|
"policy_layers": 2,
|
|
"policy_hidden": 512,
|
|
"policy_dropout": 0.0,
|
|
"learnable_cosine_scale": true
|
|
},
|
|
"parameters": 6714639,
|
|
"checkpoint_bytes": 27922475
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 27922475
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 1.5958999993017642,
|
|
"single_query_latency_ms_p50": 1.5958999993017642,
|
|
"single_query_latency_ms_p95": 1.6187000001082197,
|
|
"queries_per_second": 626.6056773215856,
|
|
"batched_qps_64": 36843.43841344889,
|
|
"batched_qps_256": 134976.95897715187,
|
|
"batched_latency_ms_64": 0.027141875000324944,
|
|
"samples": 5,
|
|
"source": "bench_router_latency.py (interleaved round-robin medians)"
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 1.4327199997448286,
|
|
"single_query_latency_ms_p50": 1.4391999975487124,
|
|
"single_query_latency_ms_p95": 1.6943999980867375,
|
|
"queries_per_second": 697.9730862821087,
|
|
"samples": 30,
|
|
"batched_qps_64": 2507.9254362757065,
|
|
"batched_latency_ms_64": 0.3987359374946209,
|
|
"batched_qps_256": 2349.3282205997198,
|
|
"batched_latency_ms_256": 0.4256535937514627
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.9518917705175705,
|
|
"recall1": 0.8963736387111261,
|
|
"recall3": 0.9745424946671157,
|
|
"recall5": 0.9820366004266308,
|
|
"recall8": 0.9912709105198159,
|
|
"evidence_count_in_top3": 1.0851577411025037,
|
|
"hit3": 0.9761984955652857,
|
|
"mrr": 0.9670023550641874,
|
|
"ndcg3": 0.9642642772784563,
|
|
"mean_first_positive_rank": 1.1905243067250477,
|
|
"all_evidence_in_top3": 0.9728864937689458,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9705,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 200.33457717921095,
|
|
"median_score_margin": 219.4529800415039,
|
|
"negative_margin_rate": 0.04810822948242955,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999979790499,
|
|
"unknown_mean_need_prob": 3.8975788264237145e-06,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 3.895038815020929e-06,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.984375,
|
|
"recall1": 0.9225625,
|
|
"recall3": 0.99815625,
|
|
"recall5": 0.9995,
|
|
"recall8": 0.99959375,
|
|
"evidence_count_in_top3": 1.1213125,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9921354166666666,
|
|
"ndcg3": 0.9914167841247224,
|
|
"mean_first_positive_rank": 1.0159375,
|
|
"all_evidence_in_top3": 0.9963125,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9705,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 217.4091286659348,
|
|
"median_score_margin": 228.10742282867432,
|
|
"negative_margin_rate": 0.015625,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999999627471,
|
|
"unknown_mean_need_prob": 6.477803048274694e-11,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.6754540128881078,
|
|
"recall1": 0.6754540128881078,
|
|
"recall3": 0.7592267135325131,
|
|
"recall5": 0.8189806678383128,
|
|
"recall8": 0.9127123608670181,
|
|
"evidence_count_in_top3": 0.7592267135325131,
|
|
"hit3": 0.7592267135325131,
|
|
"mrr": 0.7481470978152184,
|
|
"ndcg3": 0.7247037236923608,
|
|
"mean_first_positive_rank": 2.7809021675454013,
|
|
"all_evidence_in_top3": 0.7592267135325131,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 52.43232965865677,
|
|
"median_score_margin": 72.44926834106445,
|
|
"negative_margin_rate": 0.3245459871118922,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9999999792588407,
|
|
"unknown_mean_need_prob": 0.0003750932634526786,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.5046728971962616,
|
|
"recall1": 0.5046728971962616,
|
|
"recall3": 0.8785046728971962,
|
|
"recall5": 0.9719626168224299,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 0.8785046728971962,
|
|
"hit3": 0.8785046728971962,
|
|
"mrr": 0.7002447708055184,
|
|
"ndcg3": 0.7258507766355217,
|
|
"mean_first_positive_rank": 1.925233644859813,
|
|
"all_evidence_in_top3": 0.8785046728971962,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 6.655256898882233,
|
|
"median_score_margin": 1.441606044769287,
|
|
"negative_margin_rate": 0.4953271028037383,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0003750932634526786,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9945,
|
|
"recall1": 0.9945,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99725,
|
|
"ndcg3": 0.9979701136446429,
|
|
"mean_first_positive_rank": 1.0055,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 243.0819873433709,
|
|
"median_score_margin": 235.45626258850098,
|
|
"negative_margin_rate": 0.0055,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9785,
|
|
"recall1": 0.9785,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98925,
|
|
"ndcg3": 0.9920649897017864,
|
|
"mean_first_positive_rank": 1.0215,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 229.82924858646467,
|
|
"median_score_margin": 248.98048400878906,
|
|
"negative_margin_rate": 0.0215,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9845,
|
|
"recall1": 0.9845,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9921666666666666,
|
|
"ndcg3": 0.9942139463035718,
|
|
"mean_first_positive_rank": 1.016,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 232.02638172297924,
|
|
"median_score_margin": 249.2184543609619,
|
|
"negative_margin_rate": 0.0155,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 1.2955399044485864e-10,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9815,
|
|
"recall1": 0.9815,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9906666666666666,
|
|
"ndcg3": 0.9931067355642863,
|
|
"mean_first_positive_rank": 1.019,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 211.99539810842276,
|
|
"median_score_margin": 225.90464401245117,
|
|
"negative_margin_rate": 0.0185,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.989,
|
|
"recall1": 0.4945,
|
|
"recall3": 0.98525,
|
|
"recall5": 0.996,
|
|
"recall8": 0.99675,
|
|
"evidence_count_in_top3": 1.9705,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9945,
|
|
"ndcg3": 0.9737356054745618,
|
|
"mean_first_positive_rank": 1.011,
|
|
"all_evidence_in_top3": 0.9705,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9705,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 234.29880876231567,
|
|
"median_score_margin": 254.67290210723877,
|
|
"negative_margin_rate": 0.011,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999997019767,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.997,
|
|
"recall1": 0.997,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9985,
|
|
"ndcg3": 0.9988927892607143,
|
|
"mean_first_positive_rank": 1.003,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 155.67033711057155,
|
|
"median_score_margin": 155.824444770813,
|
|
"negative_margin_rate": 0.003,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.981,
|
|
"recall1": 0.981,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9905,
|
|
"ndcg3": 0.9929876653178578,
|
|
"mean_first_positive_rank": 1.019,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 217.68418576198818,
|
|
"median_score_margin": 236.54215621948242,
|
|
"negative_margin_rate": 0.019,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.969,
|
|
"recall1": 0.969,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98425,
|
|
"ndcg3": 0.988362427730358,
|
|
"mean_first_positive_rank": 1.0325,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 214.68668193136529,
|
|
"median_score_margin": 221.31105995178223,
|
|
"negative_margin_rate": 0.031,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.665380374862183,
|
|
"recall1": 0.665380374862183,
|
|
"recall3": 0.7662624035281147,
|
|
"recall5": 0.8280044101433297,
|
|
"recall8": 0.9178610804851157,
|
|
"evidence_count_in_top3": 0.7662624035281147,
|
|
"hit3": 0.7662624035281147,
|
|
"mrr": 0.745321547104062,
|
|
"ndcg3": 0.7247713833753366,
|
|
"mean_first_positive_rank": 2.7304299889746417,
|
|
"all_evidence_in_top3": 0.7662624035281147,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 49.73213848704934,
|
|
"median_score_margin": 64.13029479980469,
|
|
"negative_margin_rate": 0.33461962513781696,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9999999804822718,
|
|
"unknown_mean_need_prob": 0.0001509735806525834,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 2.0705206352511507e-15,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"XL-512 v6 best": {
|
|
"checkpoint": "checkpoints\\router_v6_xl512\\router_best.pt",
|
|
"router": {
|
|
"kind": "router_xl",
|
|
"step": 36000,
|
|
"arch_config": {
|
|
"arch": "router_xl",
|
|
"arch_version": 1,
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3,
|
|
"encoder_layers": 2,
|
|
"encoder_hidden": 512,
|
|
"pair_blocks": 1,
|
|
"pair_hidden": 512,
|
|
"pair_expansion": 2,
|
|
"pair_dropout": 0.05,
|
|
"use_interaction": true,
|
|
"policy_layers": 2,
|
|
"policy_hidden": 512,
|
|
"policy_dropout": 0.0,
|
|
"learnable_cosine_scale": true
|
|
},
|
|
"parameters": 7898127,
|
|
"checkpoint_bytes": 99014505
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048,
|
|
"checkpoint_bytes": 99014505
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 1.5855000019655563,
|
|
"single_query_latency_ms_p50": 1.5855000019655563,
|
|
"single_query_latency_ms_p95": 1.6239000015048077,
|
|
"queries_per_second": 630.7158617220388,
|
|
"batched_qps_64": 39909.70428801726,
|
|
"batched_qps_256": 94647.25411335309,
|
|
"batched_latency_ms_64": 0.025056562503777968,
|
|
"samples": 5,
|
|
"source": "bench_router_latency.py (interleaved round-robin medians)"
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 1.7938500001037028,
|
|
"single_query_latency_ms_p50": 1.812500002415618,
|
|
"single_query_latency_ms_p95": 2.0920999995723832,
|
|
"queries_per_second": 557.4602112451932,
|
|
"samples": 30,
|
|
"batched_qps_64": 1675.9990132534788,
|
|
"batched_latency_ms_64": 0.5966590625007484,
|
|
"batched_qps_256": 1750.842524579076,
|
|
"batched_latency_ms_256": 0.5711535937479084
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.957842146626249,
|
|
"recall1": 0.9024082182553048,
|
|
"recall3": 0.9758897496351184,
|
|
"recall5": 0.9834399910183003,
|
|
"recall8": 0.9905411474121477,
|
|
"evidence_count_in_top3": 1.0808914337038285,
|
|
"hit3": 0.9831593128999663,
|
|
"mrr": 0.9724377931581535,
|
|
"ndcg3": 0.9664212737428025,
|
|
"mean_first_positive_rank": 1.1337150555742674,
|
|
"all_evidence_in_top3": 0.9686201863702706,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.8705,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 45.56406442258217,
|
|
"median_score_margin": 47.89992094039917,
|
|
"negative_margin_rate": 0.042157853373750984,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999977515257,
|
|
"unknown_mean_need_prob": 9.165001448203187e-06,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 1.130629900134017e-05,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.98375,
|
|
"recall1": 0.92203125,
|
|
"recall3": 0.99190625,
|
|
"recall5": 0.994875,
|
|
"recall8": 0.99640625,
|
|
"evidence_count_in_top3": 1.1088125,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99184375,
|
|
"ndcg3": 0.9863265131728122,
|
|
"mean_first_positive_rank": 1.0164375,
|
|
"all_evidence_in_top3": 0.9838125,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.8705,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 48.28189839535998,
|
|
"median_score_margin": 49.04537320137024,
|
|
"negative_margin_rate": 0.01625,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999999552965,
|
|
"unknown_mean_need_prob": 2.862315262235959e-10,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.7457527826596367,
|
|
"recall1": 0.7457527826596367,
|
|
"recall3": 0.8347978910369068,
|
|
"recall5": 0.8763913298183948,
|
|
"recall8": 0.9355594610427651,
|
|
"evidence_count_in_top3": 0.8347978910369068,
|
|
"hit3": 0.8347978910369068,
|
|
"mrr": 0.8100220298155272,
|
|
"ndcg3": 0.7979455040170743,
|
|
"mean_first_positive_rank": 2.16813122437024,
|
|
"all_evidence_in_top3": 0.8347978910369068,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 22.8850553075215,
|
|
"median_score_margin": 27.899999618530273,
|
|
"negative_margin_rate": 0.2542472173403632,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9999999769542673,
|
|
"unknown_mean_need_prob": 0.0008787320924793242,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.4672897196261682,
|
|
"recall1": 0.4672897196261682,
|
|
"recall3": 0.8317757009345794,
|
|
"recall5": 0.9813084112149533,
|
|
"recall8": 0.9906542056074766,
|
|
"evidence_count_in_top3": 0.8317757009345794,
|
|
"hit3": 0.8317757009345794,
|
|
"mrr": 0.6616751628433871,
|
|
"ndcg3": 0.6776764890854534,
|
|
"mean_first_positive_rank": 2.1682242990654204,
|
|
"all_evidence_in_top3": 0.8317757009345794,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.9633634409336286,
|
|
"median_score_margin": -1.062392234802246,
|
|
"negative_margin_rate": 0.5327102803738317,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0008787320924793242,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.992,
|
|
"recall1": 0.992,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.996,
|
|
"ndcg3": 0.9970474380285717,
|
|
"mean_first_positive_rank": 1.008,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 67.69295654568822,
|
|
"median_score_margin": 69.3216142654419,
|
|
"negative_margin_rate": 0.008,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9795,
|
|
"recall1": 0.9795,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98975,
|
|
"ndcg3": 0.9924340599482149,
|
|
"mean_first_positive_rank": 1.0205,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 39.98083441020921,
|
|
"median_score_margin": 44.252100229263306,
|
|
"negative_margin_rate": 0.0205,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9805,
|
|
"recall1": 0.9805,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99025,
|
|
"ndcg3": 0.9928031301946435,
|
|
"mean_first_positive_rank": 1.0195,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 40.70788796337694,
|
|
"median_score_margin": 44.594672441482544,
|
|
"negative_margin_rate": 0.0195,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 5.724454182644648e-10,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9815,
|
|
"recall1": 0.9815,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9906666666666666,
|
|
"ndcg3": 0.9931067355642863,
|
|
"mean_first_positive_rank": 1.019,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 37.93500039816182,
|
|
"median_score_margin": 40.258564949035645,
|
|
"negative_margin_rate": 0.0185,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9875,
|
|
"recall1": 0.49375,
|
|
"recall3": 0.93525,
|
|
"recall5": 0.959,
|
|
"recall8": 0.97125,
|
|
"evidence_count_in_top3": 1.8705,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9936666666666666,
|
|
"ndcg3": 0.9341087890914226,
|
|
"mean_first_positive_rank": 1.013,
|
|
"all_evidence_in_top3": 0.8705,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.8705,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 47.13378771021217,
|
|
"median_score_margin": 51.072168588638306,
|
|
"negative_margin_rate": 0.0125,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999996423722,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9965,
|
|
"recall1": 0.9965,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99825,
|
|
"ndcg3": 0.9987082541375002,
|
|
"mean_first_positive_rank": 1.0035,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 61.5167154010348,
|
|
"median_score_margin": 62.94392251968384,
|
|
"negative_margin_rate": 0.0035,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9815,
|
|
"recall1": 0.9815,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99075,
|
|
"ndcg3": 0.9931722004410719,
|
|
"mean_first_positive_rank": 1.0185,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 38.85421975574363,
|
|
"median_score_margin": 43.87380111217499,
|
|
"negative_margin_rate": 0.0185,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.971,
|
|
"recall1": 0.971,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9854166666666666,
|
|
"ndcg3": 0.9892314979767864,
|
|
"mean_first_positive_rank": 1.0295,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 52.43378497845307,
|
|
"median_score_margin": 59.86355948448181,
|
|
"negative_margin_rate": 0.029,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.7293274531422271,
|
|
"recall1": 0.7293274531422271,
|
|
"recall3": 0.834619625137817,
|
|
"recall5": 0.88257993384785,
|
|
"recall8": 0.9388092613009923,
|
|
"evidence_count_in_top3": 0.834619625137817,
|
|
"hit3": 0.834619625137817,
|
|
"mrr": 0.8012716909147449,
|
|
"ndcg3": 0.7908513559477891,
|
|
"mean_first_positive_rank": 2.1681367144432193,
|
|
"all_evidence_in_top3": 0.834619625137817,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 21.591989690253087,
|
|
"median_score_margin": 24.97784423828125,
|
|
"negative_margin_rate": 0.27067254685777287,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9999999783136353,
|
|
"unknown_mean_need_prob": 0.0003550033115114848,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 1.7634182726834057e-14,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"V2-512 v6 best": {
|
|
"checkpoint": "checkpoints\\router_v6_v2_512\\router_best.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": 2000,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 4741902,
|
|
"checkpoint_bytes": 56922573
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048,
|
|
"checkpoint_bytes": 56922573
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 1.162250000561471,
|
|
"single_query_latency_ms_p50": 1.162250000561471,
|
|
"single_query_latency_ms_p95": 1.1778999996749917,
|
|
"queries_per_second": 860.4000856243581,
|
|
"batched_qps_64": 53757.11860745415,
|
|
"batched_qps_256": 154085.05975293368,
|
|
"batched_latency_ms_64": 0.01860218750380227,
|
|
"samples": 5,
|
|
"source": "bench_router_latency.py (interleaved round-robin medians)"
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.8890533339581452,
|
|
"single_query_latency_ms_p50": 0.8698999990883749,
|
|
"single_query_latency_ms_p95": 1.0765000006358605,
|
|
"queries_per_second": 1124.7919127055181,
|
|
"samples": 30,
|
|
"batched_qps_64": 3131.6700902508155,
|
|
"batched_latency_ms_64": 0.31931843750498956,
|
|
"batched_qps_256": 3247.1683677121205,
|
|
"batched_latency_ms_256": 0.30796062499973686
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.950488379925901,
|
|
"recall1": 0.8971034018187942,
|
|
"recall3": 0.9884079937128102,
|
|
"recall5": 0.9934882676546536,
|
|
"recall8": 0.9961266419669923,
|
|
"evidence_count_in_top3": 1.0986302907825305,
|
|
"hit3": 0.9904569439766476,
|
|
"mrr": 0.9711571604054701,
|
|
"ndcg3": 0.9723689530227545,
|
|
"mean_first_positive_rank": 1.1156393847535646,
|
|
"all_evidence_in_top3": 0.9863590434489727,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9635,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 12.36926158705623,
|
|
"median_score_margin": 12.002254128456116,
|
|
"negative_margin_rate": 0.049511620074099025,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9988408471058552,
|
|
"unknown_mean_need_prob": 0.006478862109410833,
|
|
"thr0.30": {
|
|
"need_precision": 0.999326825984517,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9996632996632996,
|
|
"specificity_unknown_refusal": 0.9970774476376035,
|
|
"abstention_accuracy": 0.9994525547445255,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0029225523623964927,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4094,
|
|
"fp": 12,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.999326825984517,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9996632996632996,
|
|
"specificity_unknown_refusal": 0.9970774476376035,
|
|
"abstention_accuracy": 0.9994525547445255,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0029225523623964927,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4094,
|
|
"fp": 12,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.9995511165974638,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.999775507913346,
|
|
"specificity_unknown_refusal": 0.9980516317584024,
|
|
"abstention_accuracy": 0.9996350364963503,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.001948368241597662,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4098,
|
|
"fp": 8,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.9998877413560844,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9999438675273646,
|
|
"specificity_unknown_refusal": 0.9995129079396006,
|
|
"abstention_accuracy": 0.9999087591240876,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0004870920603994155,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4104,
|
|
"fp": 2,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.0904978112084791,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.97,
|
|
"recall1": 0.9105625,
|
|
"recall3": 0.99771875,
|
|
"recall5": 0.9999375,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.1204375,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9849166666666667,
|
|
"ndcg3": 0.9857856193167982,
|
|
"mean_first_positive_rank": 1.0305,
|
|
"all_evidence_in_top3": 0.9954375,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9635,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 13.542529965203517,
|
|
"median_score_margin": 12.727105312049389,
|
|
"negative_margin_rate": 0.03,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9990760059095919,
|
|
"unknown_mean_need_prob": 0.0025661538663334796,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.8125366139425894,
|
|
"recall1": 0.8125366139425894,
|
|
"recall3": 0.9244288224956063,
|
|
"recall5": 0.9502050380785003,
|
|
"recall8": 0.971294669009959,
|
|
"evidence_count_in_top3": 0.9244288224956063,
|
|
"hit3": 0.9244288224956063,
|
|
"mrr": 0.8742941912651278,
|
|
"ndcg3": 0.878837443896981,
|
|
"mean_first_positive_rank": 1.7176332747510252,
|
|
"all_evidence_in_top3": 0.9244288224956063,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 2.2332465364638656,
|
|
"median_score_margin": 2.495278835296631,
|
|
"negative_margin_rate": 0.18746338605741067,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.996912036259131,
|
|
"unknown_mean_need_prob": 0.2510888675848643,
|
|
"thr0.50": {
|
|
"need_precision": 0.9976621858562245,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9988297249853716,
|
|
"specificity_unknown_refusal": 0.8095238095238095,
|
|
"abstention_accuracy": 0.9976851851851852,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.19047619047619047,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 17,
|
|
"fp": 4,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.2336448598130841,
|
|
"recall1": 0.2336448598130841,
|
|
"recall3": 0.616822429906542,
|
|
"recall5": 0.719626168224299,
|
|
"recall8": 0.8130841121495327,
|
|
"evidence_count_in_top3": 0.616822429906542,
|
|
"hit3": 0.616822429906542,
|
|
"mrr": 0.4589420963252739,
|
|
"ndcg3": 0.4582719938918631,
|
|
"mean_first_positive_rank": 4.242990654205608,
|
|
"all_evidence_in_top3": 0.616822429906542,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -1.3701436372003823,
|
|
"median_score_margin": -0.6190404891967773,
|
|
"negative_margin_rate": 0.7663551401869159,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 0.9944477560364198,
|
|
"unknown_mean_need_prob": 0.2510888675848643,
|
|
"thr0.50": {
|
|
"need_precision": 0.963963963963964,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9816513761467891,
|
|
"specificity_unknown_refusal": 0.8095238095238095,
|
|
"abstention_accuracy": 0.96875,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.19047619047619047,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 17,
|
|
"fp": 4,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9905,
|
|
"recall1": 0.9905,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99525,
|
|
"ndcg3": 0.9964938326589288,
|
|
"mean_first_positive_rank": 1.0095,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 16.784344019554556,
|
|
"median_score_margin": 14.332054734230042,
|
|
"negative_margin_rate": 0.0095,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9984212898612023,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.974,
|
|
"recall1": 0.974,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.987,
|
|
"ndcg3": 0.9904041735928579,
|
|
"mean_first_positive_rank": 1.026,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 14.32004562981613,
|
|
"median_score_margin": 12.83707270026207,
|
|
"negative_margin_rate": 0.026,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9998619016408921,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.969,
|
|
"recall1": 0.969,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9844166666666666,
|
|
"ndcg3": 0.9884933574839294,
|
|
"mean_first_positive_rank": 1.0315,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 14.515158921549562,
|
|
"median_score_margin": 12.90774929523468,
|
|
"negative_margin_rate": 0.031,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9998633873462677,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.004985218498797622,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.973,
|
|
"recall1": 0.973,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9864166666666666,
|
|
"ndcg3": 0.9899696384696437,
|
|
"mean_first_positive_rank": 1.0275,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 11.82059130709432,
|
|
"median_score_margin": 12.446841090917587,
|
|
"negative_margin_rate": 0.027,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9998804149031639,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.951,
|
|
"recall1": 0.4755,
|
|
"recall3": 0.98175,
|
|
"recall5": 0.9995,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.9635,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9753333333333334,
|
|
"ndcg3": 0.9571701608629515,
|
|
"mean_first_positive_rank": 1.05,
|
|
"all_evidence_in_top3": 0.9635,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9635,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 10.0001081677787,
|
|
"median_score_margin": 11.238629460334778,
|
|
"negative_margin_rate": 0.049,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9978087923228741,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9755,
|
|
"recall1": 0.9755,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98775,
|
|
"ndcg3": 0.9909577789625007,
|
|
"mean_first_positive_rank": 1.0245,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 14.530858453191817,
|
|
"median_score_margin": 12.788989305496216,
|
|
"negative_margin_rate": 0.0245,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9972889910042286,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9745,
|
|
"recall1": 0.9745,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9871666666666666,
|
|
"ndcg3": 0.9905232438392865,
|
|
"mean_first_positive_rank": 1.026,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 13.56638970718137,
|
|
"median_score_margin": 13.154640913009644,
|
|
"negative_margin_rate": 0.0255,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9995595035254955,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9525,
|
|
"recall1": 0.9525,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.976,
|
|
"ndcg3": 0.982272768664287,
|
|
"mean_first_positive_rank": 1.049,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 12.802743515461684,
|
|
"median_score_margin": 12.365995168685913,
|
|
"negative_margin_rate": 0.0475,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999237666726113,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.7783902976846747,
|
|
"recall1": 0.7783902976846747,
|
|
"recall3": 0.906284454244763,
|
|
"recall5": 0.9366041896361632,
|
|
"recall8": 0.961962513781698,
|
|
"evidence_count_in_top3": 0.906284454244763,
|
|
"hit3": 0.906284454244763,
|
|
"mrr": 0.8497943708910569,
|
|
"ndcg3": 0.8540301102968997,
|
|
"mean_first_positive_rank": 1.866593164277839,
|
|
"all_evidence_in_top3": 0.906284454244763,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 2.0206981634858754,
|
|
"median_score_margin": 2.3058557510375977,
|
|
"negative_margin_rate": 0.22160970231532526,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9967666790464352,
|
|
"unknown_mean_need_prob": 0.1541282297727072,
|
|
"thr0.50": {
|
|
"need_precision": 0.995609220636663,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9977997799779977,
|
|
"specificity_unknown_refusal": 0.9245283018867925,
|
|
"abstention_accuracy": 0.9958333333333333,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.07547169811320754,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 98,
|
|
"fp": 8,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.00014708923386933748,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} |