WpyQwq
643e22ecb9
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线 - 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同） - 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00） - 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00% - 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益） - 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
6124 lines
196 KiB
JSON
6124 lines
196 KiB
JSON
{
|
|
"V2-128 deployed(v3)": {
|
|
"checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_qwen_router_entities\\memory_router_v2.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 2037774,
|
|
"checkpoint_bytes": 8157561
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 8157561
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 0.8439246668907193,
|
|
"single_query_latency_ms_p50": 0.8266000004368834,
|
|
"single_query_latency_ms_p95": 1.1234000012336764,
|
|
"queries_per_second": 1184.9398876848934,
|
|
"samples": 150,
|
|
"batched_qps_64": 72916.19191648123,
|
|
"batched_latency_ms_64": 0.013714375006657065,
|
|
"batched_qps_256": 264812.97583671653,
|
|
"batched_latency_ms_256": 0.003776249999987158
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.3935313334174377,
|
|
"single_query_latency_ms_p50": 0.3775000004679896,
|
|
"single_query_latency_ms_p95": 0.5684999996447004,
|
|
"queries_per_second": 2541.093720075529,
|
|
"samples": 150,
|
|
"batched_qps_64": 11881.908679212764,
|
|
"batched_latency_ms_64": 0.08416156250632412,
|
|
"batched_qps_256": 13117.630300343435,
|
|
"batched_latency_ms_256": 0.07623328124850559
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.4112495789828225,
|
|
"recall1": 0.37027057370607386,
|
|
"recall3": 0.4673009992141013,
|
|
"recall5": 0.4891377568204783,
|
|
"recall8": 0.5160267205568654,
|
|
"evidence_count_in_top3": 0.5420455821264174,
|
|
"hit3": 0.5003368137420007,
|
|
"mrr": 0.476908110965675,
|
|
"ndcg3": 0.44220238482445295,
|
|
"mean_first_positive_rank": 14.38671831144044,
|
|
"all_evidence_in_top3": 0.43426518468620184,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.3715,
|
|
"hop_accuracy": 0.7322992700729927,
|
|
"hop_accuracy_answerable": 0.9010890310991355,
|
|
"hop_under_prediction": 0.046817110138093636,
|
|
"hop_over_prediction": 0.05209385876277085,
|
|
"mean_score_margin": -9.29892828391031,
|
|
"median_score_margin": -9.269989609718323,
|
|
"negative_margin_rate": 0.5887504210171774,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 4042,
|
|
"true_0_pred_2": 64,
|
|
"true_1_pred_0": 834,
|
|
"true_1_pred_1": 14052,
|
|
"true_1_pred_2": 928,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9762653562039442,
|
|
"unknown_mean_need_prob": 0.998855464207621,
|
|
"thr0.30": {
|
|
"need_precision": 0.8126824817518248,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.8966628076710123,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8126824817518248,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.8126824817518248,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.8966628076710123,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8126824817518248,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.8124086257309941,
|
|
"need_recall": 0.998203660042663,
|
|
"need_f1": 0.895773512669387,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8112226277372263,
|
|
"known_question_refusal_rate": 0.001796339957336926,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17782,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 32
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.8115476409032495,
|
|
"need_recall": 0.9925900976759852,
|
|
"need_f1": 0.892985202767537,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8066605839416059,
|
|
"known_question_refusal_rate": 0.00740990232401482,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17682,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 132
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 0.8112270700197692,
|
|
"need_recall": 0.9905130796003143,
|
|
"need_f1": 0.8919499557689878,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8049726277372263,
|
|
"known_question_refusal_rate": 0.00948692039968564,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17645,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 169
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 0.8106525247867189,
|
|
"need_recall": 0.9868081284383069,
|
|
"need_f1": 0.8900984835059115,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8019616788321168,
|
|
"known_question_refusal_rate": 0.01319187156169305,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 17579,
|
|
"tn": 0,
|
|
"fp": 4106,
|
|
"fn": 235
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_2": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.9999341862276196,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 64,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.442375,
|
|
"recall1": 0.39675,
|
|
"recall3": 0.49153125,
|
|
"recall5": 0.50303125,
|
|
"recall8": 0.51234375,
|
|
"evidence_count_in_top3": 0.57475,
|
|
"hit3": 0.5283125,
|
|
"mrr": 0.500402578720693,
|
|
"ndcg3": 0.4692792057097287,
|
|
"mean_first_positive_rank": 14.9525,
|
|
"all_evidence_in_top3": 0.45475,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.3715,
|
|
"hop_accuracy": 0.74955,
|
|
"hop_accuracy_answerable": 0.9369375,
|
|
"hop_under_prediction": 0.0160625,
|
|
"hop_over_prediction": 0.047,
|
|
"mean_score_margin": -8.091850390457548,
|
|
"median_score_margin": -6.662378132343292,
|
|
"negative_margin_rate": 0.557625,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 4000,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 257,
|
|
"true_1_pred_1": 12991,
|
|
"true_1_pred_2": 752,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9821553228273988,
|
|
"unknown_mean_need_prob": 0.9988265322595835,
|
|
"thr0.50": {
|
|
"need_precision": 0.8,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.888888888888889,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 0,
|
|
"fp": 4000,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.14528412419449327,
|
|
"recall1": 0.14528412419449327,
|
|
"recall3": 0.2677211482132396,
|
|
"recall5": 0.38488576449912126,
|
|
"recall8": 0.5653192735793791,
|
|
"evidence_count_in_top3": 0.2677211482132396,
|
|
"hit3": 0.2677211482132396,
|
|
"mrr": 0.2801716446629072,
|
|
"ndcg3": 0.21516992510461316,
|
|
"mean_first_positive_rank": 9.195664909197422,
|
|
"all_evidence_in_top3": 0.2677211482132396,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.5740740740740741,
|
|
"hop_accuracy_answerable": 0.5811364967779731,
|
|
"hop_under_prediction": 0.3380199179847686,
|
|
"hop_over_prediction": 0.08084358523725835,
|
|
"mean_score_margin": -19.359866383266713,
|
|
"median_score_margin": -18.159542322158813,
|
|
"negative_margin_rate": 0.8547158758055068,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 21,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 577,
|
|
"true_1_pred_1": 992,
|
|
"true_1_pred_2": 138
|
|
},
|
|
"known_mean_need_prob": 0.9195722371566764,
|
|
"unknown_mean_need_prob": 0.9999671209426153,
|
|
"thr0.50": {
|
|
"need_precision": 0.9876179245283019,
|
|
"need_recall": 0.981253661394259,
|
|
"need_f1": 0.9844255069056715,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.9693287037037037,
|
|
"known_question_refusal_rate": 0.018746338605741066,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 1675,
|
|
"tn": 0,
|
|
"fp": 21,
|
|
"fn": 32
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.028037383177570093,
|
|
"recall5": 0.07476635514018691,
|
|
"recall8": 0.2803738317757009,
|
|
"evidence_count_in_top3": 0.028037383177570093,
|
|
"hit3": 0.028037383177570093,
|
|
"mrr": 0.1023068389893556,
|
|
"ndcg3": 0.01524233414552764,
|
|
"mean_first_positive_rank": 12.598130841121495,
|
|
"all_evidence_in_top3": 0.028037383177570093,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.5390625,
|
|
"hop_accuracy_answerable": 0.6448598130841121,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.35514018691588783,
|
|
"mean_score_margin": -29.291684916086286,
|
|
"median_score_margin": -29.07153034210205,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 21,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 69,
|
|
"true_1_pred_2": 38
|
|
},
|
|
"known_mean_need_prob": 0.9999633771236812,
|
|
"unknown_mean_need_prob": 0.9999671209426153,
|
|
"thr0.50": {
|
|
"need_precision": 0.8359375,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9106382978723404,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8359375,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 0,
|
|
"fp": 21,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.796,
|
|
"recall1": 0.796,
|
|
"recall3": 0.9125,
|
|
"recall5": 0.919,
|
|
"recall8": 0.927,
|
|
"evidence_count_in_top3": 0.9125,
|
|
"hit3": 0.9125,
|
|
"mrr": 0.8582544951434297,
|
|
"ndcg3": 0.8678012294946459,
|
|
"mean_first_positive_rank": 2.8275,
|
|
"all_evidence_in_top3": 0.9125,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.4955,
|
|
"hop_accuracy_answerable": 0.4955,
|
|
"hop_under_prediction": 0.1285,
|
|
"hop_over_prediction": 0.376,
|
|
"mean_score_margin": 21.036133953779935,
|
|
"median_score_margin": 21.400100469589233,
|
|
"negative_margin_rate": 0.204,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 257,
|
|
"true_1_pred_1": 991,
|
|
"true_1_pred_2": 752
|
|
},
|
|
"known_mean_need_prob": 0.9764632563889026,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.231,
|
|
"recall1": 0.231,
|
|
"recall3": 0.2535,
|
|
"recall5": 0.255,
|
|
"recall8": 0.257,
|
|
"evidence_count_in_top3": 0.2535,
|
|
"hit3": 0.2535,
|
|
"mrr": 0.26603145346142226,
|
|
"ndcg3": 0.24486859507142916,
|
|
"mean_first_positive_rank": 23.789,
|
|
"all_evidence_in_top3": 0.2535,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -27.718347060799598,
|
|
"median_score_margin": -38.59446668624878,
|
|
"negative_margin_rate": 0.769,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9723067943453789,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.2255,
|
|
"recall1": 0.2255,
|
|
"recall3": 0.2455,
|
|
"recall5": 0.2475,
|
|
"recall8": 0.2485,
|
|
"evidence_count_in_top3": 0.2455,
|
|
"hit3": 0.2455,
|
|
"mrr": 0.2596338049221357,
|
|
"ndcg3": 0.23792220044107196,
|
|
"mean_first_positive_rank": 24.0695,
|
|
"all_evidence_in_top3": 0.2455,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -28.903205164782705,
|
|
"median_score_margin": -39.01692509651184,
|
|
"negative_margin_rate": 0.7745,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9724746745526791,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.99774328109622,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 2000,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5255,
|
|
"recall1": 0.5255,
|
|
"recall3": 0.6245,
|
|
"recall5": 0.6265,
|
|
"recall8": 0.6295,
|
|
"evidence_count_in_top3": 0.6245,
|
|
"hit3": 0.6245,
|
|
"mrr": 0.5855216777342396,
|
|
"ndcg3": 0.585867169546431,
|
|
"mean_first_positive_rank": 12.2085,
|
|
"all_evidence_in_top3": 0.6245,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -1.0039909677058458,
|
|
"median_score_margin": 2.929476022720337,
|
|
"negative_margin_rate": 0.4745,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9875293919146061,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.73,
|
|
"recall1": 0.365,
|
|
"recall3": 0.66575,
|
|
"recall5": 0.72725,
|
|
"recall8": 0.77375,
|
|
"evidence_count_in_top3": 1.3315,
|
|
"hit3": 0.96,
|
|
"mrr": 0.84532448592014,
|
|
"ndcg3": 0.6658792959224611,
|
|
"mean_first_positive_rank": 1.59,
|
|
"all_evidence_in_top3": 0.3715,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.3715,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 2.786984693795443,
|
|
"median_score_margin": 5.27826452255249,
|
|
"negative_margin_rate": 0.27,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9993210740983486,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.443,
|
|
"recall1": 0.443,
|
|
"recall3": 0.5525,
|
|
"recall5": 0.5615,
|
|
"recall8": 0.5665,
|
|
"evidence_count_in_top3": 0.5525,
|
|
"hit3": 0.5525,
|
|
"mrr": 0.5118924660255056,
|
|
"ndcg3": 0.5092063534375025,
|
|
"mean_first_positive_rank": 13.8255,
|
|
"all_evidence_in_top3": 0.5525,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -2.995655853047967,
|
|
"median_score_margin": -5.5251545906066895,
|
|
"negative_margin_rate": 0.557,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9880377069413662,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.249,
|
|
"recall1": 0.249,
|
|
"recall3": 0.287,
|
|
"recall5": 0.293,
|
|
"recall8": 0.3005,
|
|
"evidence_count_in_top3": 0.287,
|
|
"hit3": 0.287,
|
|
"mrr": 0.2919631936667751,
|
|
"ndcg3": 0.27173149797678653,
|
|
"mean_first_positive_rank": 22.067,
|
|
"all_evidence_in_top3": 0.287,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -19.72415616109967,
|
|
"median_score_margin": -21.928287506103516,
|
|
"negative_margin_rate": 0.751,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9656237106621265,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.339,
|
|
"recall1": 0.339,
|
|
"recall3": 0.391,
|
|
"recall5": 0.3945,
|
|
"recall8": 0.396,
|
|
"evidence_count_in_top3": 0.391,
|
|
"hit3": 0.391,
|
|
"mrr": 0.3845990528918967,
|
|
"ndcg3": 0.3709573037875013,
|
|
"mean_first_positive_rank": 19.243,
|
|
"all_evidence_in_top3": 0.391,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -8.212566563799978,
|
|
"median_score_margin": -16.937832549214363,
|
|
"negative_margin_rate": 0.661,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9954859737157822,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.13671444321940462,
|
|
"recall1": 0.13671444321940462,
|
|
"recall3": 0.2535832414553473,
|
|
"recall5": 0.36659316427783906,
|
|
"recall8": 0.5485115766262404,
|
|
"evidence_count_in_top3": 0.2535832414553473,
|
|
"hit3": 0.2535832414553473,
|
|
"mrr": 0.26968017045834825,
|
|
"ndcg3": 0.2033770627933551,
|
|
"mean_first_positive_rank": 9.396361631753033,
|
|
"all_evidence_in_top3": 0.2535832414553473,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.5526041666666667,
|
|
"hop_accuracy_answerable": 0.5848952590959207,
|
|
"hop_under_prediction": 0.31808158765159866,
|
|
"hop_over_prediction": 0.0970231532524807,
|
|
"mean_score_margin": -19.945701324287494,
|
|
"median_score_margin": -18.862584590911865,
|
|
"negative_margin_rate": 0.8632855567805954,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 42,
|
|
"true_0_pred_2": 64,
|
|
"true_1_pred_0": 577,
|
|
"true_1_pred_1": 1061,
|
|
"true_1_pred_2": 176
|
|
},
|
|
"known_mean_need_prob": 0.9243141621712682,
|
|
"unknown_mean_need_prob": 0.9999472358316746,
|
|
"thr0.50": {
|
|
"need_precision": 0.9438559322033898,
|
|
"need_recall": 0.9823594266813671,
|
|
"need_f1": 0.9627228525121555,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.928125,
|
|
"known_question_refusal_rate": 0.017640573318632856,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 1782,
|
|
"tn": 0,
|
|
"fp": 106,
|
|
"fn": 32
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.9999097834229469,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 2000,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"V2-512 v3 best": {
|
|
"checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_router_512\\router_best.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": 1000,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 4741902,
|
|
"checkpoint_bytes": 56922381
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048,
|
|
"checkpoint_bytes": 56922381
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 0.8789160000742413,
|
|
"single_query_latency_ms_p50": 0.7994000006874558,
|
|
"single_query_latency_ms_p95": 1.517499997135019,
|
|
"queries_per_second": 1137.7651560735392,
|
|
"samples": 150,
|
|
"batched_qps_64": 69808.02789170416,
|
|
"batched_latency_ms_64": 0.014325000006465416,
|
|
"batched_qps_256": 167235.0043735693,
|
|
"batched_latency_ms_256": 0.005979609375117434
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.8516033333095644,
|
|
"single_query_latency_ms_p50": 0.8521000017935876,
|
|
"single_query_latency_ms_p95": 1.0607999975036364,
|
|
"queries_per_second": 1174.2556198244615,
|
|
"samples": 150,
|
|
"batched_qps_64": 3215.094870427158,
|
|
"batched_latency_ms_64": 0.3110328124989792,
|
|
"batched_qps_256": 3174.0694397097395,
|
|
"batched_latency_ms_256": 0.31505296875025124
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.807510946446615,
|
|
"recall1": 0.7698439429662064,
|
|
"recall3": 0.9501515661839003,
|
|
"recall5": 0.9730829684517796,
|
|
"recall8": 0.984506567867969,
|
|
"evidence_count_in_top3": 1.052542943752105,
|
|
"hit3": 0.9584035028629168,
|
|
"mrr": 0.8845870538857731,
|
|
"ndcg3": 0.8953026178367461,
|
|
"mean_first_positive_rank": 1.6810935219490288,
|
|
"all_evidence_in_top3": 0.9418996295048838,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.8385,
|
|
"hop_accuracy": 0.8158759124087591,
|
|
"hop_accuracy_answerable": 0.8877287526664421,
|
|
"hop_under_prediction": 0.11227124733355788,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 9.638220129697089,
|
|
"median_score_margin": 7.411673903465271,
|
|
"negative_margin_rate": 0.19248905355338497,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2070,
|
|
"true_0_pred_1": 2036,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 2000,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.8877228112198915,
|
|
"unknown_mean_need_prob": 0.6232877522355414,
|
|
"thr0.30": {
|
|
"need_precision": 0.8045772861057546,
|
|
"need_recall": 0.8900303132367801,
|
|
"need_f1": 0.8451492537313433,
|
|
"specificity_unknown_refusal": 0.06210423770092548,
|
|
"abstention_accuracy": 0.7349452554744526,
|
|
"known_question_refusal_rate": 0.10996968676321994,
|
|
"unknown_question_read_rate": 0.9378957622990746,
|
|
"counts": {
|
|
"tp": 15855,
|
|
"tn": 255,
|
|
"fp": 3851,
|
|
"fn": 1959
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.885845516159749,
|
|
"need_recall": 0.8877848882901089,
|
|
"need_f1": 0.8868141419239074,
|
|
"specificity_unknown_refusal": 0.5036531904529956,
|
|
"abstention_accuracy": 0.8158302919708029,
|
|
"known_question_refusal_rate": 0.1122151117098911,
|
|
"unknown_question_read_rate": 0.4963468095470044,
|
|
"counts": {
|
|
"tp": 15815,
|
|
"tn": 2068,
|
|
"fp": 2038,
|
|
"fn": 1999
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.8859383753501401,
|
|
"need_recall": 0.8877287526664421,
|
|
"need_f1": 0.8868326603858232,
|
|
"specificity_unknown_refusal": 0.504140282513395,
|
|
"abstention_accuracy": 0.8158759124087591,
|
|
"known_question_refusal_rate": 0.11227124733355788,
|
|
"unknown_question_read_rate": 0.49585971748660496,
|
|
"counts": {
|
|
"tp": 15814,
|
|
"tn": 2070,
|
|
"fp": 2036,
|
|
"fn": 2000
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.8868326603858232,
|
|
"need_recall": 0.8877287526664421,
|
|
"need_f1": 0.8872804802782921,
|
|
"specificity_unknown_refusal": 0.5085241110569898,
|
|
"abstention_accuracy": 0.8166970802919709,
|
|
"known_question_refusal_rate": 0.11227124733355788,
|
|
"unknown_question_read_rate": 0.49147588894301025,
|
|
"counts": {
|
|
"tp": 15814,
|
|
"tn": 2088,
|
|
"fp": 2018,
|
|
"fn": 2000
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 0.8876290974405029,
|
|
"need_recall": 0.8877287526664421,
|
|
"need_f1": 0.8876789222565254,
|
|
"specificity_unknown_refusal": 0.5124208475401851,
|
|
"abstention_accuracy": 0.8174270072992701,
|
|
"known_question_refusal_rate": 0.11227124733355788,
|
|
"unknown_question_read_rate": 0.4875791524598149,
|
|
"counts": {
|
|
"tp": 15814,
|
|
"tn": 2104,
|
|
"fp": 2002,
|
|
"fn": 2000
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 0.9022136010953903,
|
|
"need_recall": 0.8877287526664421,
|
|
"need_f1": 0.8949125686152454,
|
|
"specificity_unknown_refusal": 0.5825621042377009,
|
|
"abstention_accuracy": 0.8305656934306569,
|
|
"known_question_refusal_rate": 0.11227124733355788,
|
|
"unknown_question_read_rate": 0.41743789576229906,
|
|
"counts": {
|
|
"tp": 15814,
|
|
"tn": 2392,
|
|
"fp": 1714,
|
|
"fn": 2000
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.04609098758373875,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.8444375,
|
|
"recall1": 0.8025,
|
|
"recall3": 0.9736875,
|
|
"recall5": 0.98640625,
|
|
"recall8": 0.986875,
|
|
"evidence_count_in_top3": 1.0876875,
|
|
"hit3": 0.982875,
|
|
"mrr": 0.9118529559094745,
|
|
"ndcg3": 0.9247008759665758,
|
|
"mean_first_positive_rank": 1.5631875,
|
|
"all_evidence_in_top3": 0.9645,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.8385,
|
|
"hop_accuracy": 0.8,
|
|
"hop_accuracy_answerable": 0.875,
|
|
"hop_under_prediction": 0.125,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 10.705225070305634,
|
|
"median_score_margin": 8.861685633659363,
|
|
"negative_margin_rate": 0.1555625,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000,
|
|
"true_0_pred_1": 2000,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 2000,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.8765438592466526,
|
|
"unknown_mean_need_prob": 0.6330925791561604,
|
|
"thr0.50": {
|
|
"need_precision": 0.875,
|
|
"need_recall": 0.875,
|
|
"need_f1": 0.875,
|
|
"specificity_unknown_refusal": 0.5,
|
|
"abstention_accuracy": 0.8,
|
|
"known_question_refusal_rate": 0.125,
|
|
"unknown_question_read_rate": 0.5,
|
|
"counts": {
|
|
"tp": 14000,
|
|
"tn": 2000,
|
|
"fp": 2000,
|
|
"fn": 2000
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.4856473345049795,
|
|
"recall1": 0.4856473345049795,
|
|
"recall3": 0.7375512595196251,
|
|
"recall5": 0.8482718219097832,
|
|
"recall8": 0.961335676625659,
|
|
"evidence_count_in_top3": 0.7375512595196251,
|
|
"hit3": 0.7375512595196251,
|
|
"mrr": 0.6442412517298798,
|
|
"ndcg3": 0.6345330968470215,
|
|
"mean_first_positive_rank": 2.754540128881078,
|
|
"all_evidence_in_top3": 0.7375512595196251,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.9895833333333334,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.24317214623591182,
|
|
"median_score_margin": -0.0259857177734375,
|
|
"negative_margin_rate": 0.5143526654950205,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 3,
|
|
"true_0_pred_1": 18,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9860711831339773,
|
|
"unknown_mean_need_prob": 0.5690326392650604,
|
|
"thr0.50": {
|
|
"need_precision": 0.9895652173913043,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9947552447552448,
|
|
"specificity_unknown_refusal": 0.14285714285714285,
|
|
"abstention_accuracy": 0.9895833333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.8571428571428571,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 3,
|
|
"fp": 18,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.4205607476635514,
|
|
"recall1": 0.4205607476635514,
|
|
"recall3": 0.822429906542056,
|
|
"recall5": 0.9719626168224299,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 0.822429906542056,
|
|
"hit3": 0.822429906542056,
|
|
"mrr": 0.6417445482866044,
|
|
"ndcg3": 0.6594282463618242,
|
|
"mean_first_positive_rank": 2.1869158878504673,
|
|
"all_evidence_in_top3": 0.822429906542056,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.859375,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.03217372047566922,
|
|
"median_score_margin": -0.08555316925048828,
|
|
"negative_margin_rate": 0.5794392523364486,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 3,
|
|
"true_0_pred_1": 18,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 0.9903635655608133,
|
|
"unknown_mean_need_prob": 0.5690326392650604,
|
|
"thr0.50": {
|
|
"need_precision": 0.856,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9224137931034483,
|
|
"specificity_unknown_refusal": 0.14285714285714285,
|
|
"abstention_accuracy": 0.859375,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.8571428571428571,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 3,
|
|
"fp": 18,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.8605,
|
|
"recall1": 0.8605,
|
|
"recall3": 0.9995,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 0.9995,
|
|
"hit3": 0.9995,
|
|
"mrr": 0.9282083333333334,
|
|
"ndcg3": 0.9466935435803608,
|
|
"mean_first_positive_rank": 1.152,
|
|
"all_evidence_in_top3": 0.9995,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 11.369050871308893,
|
|
"median_score_margin": 8.467366218566895,
|
|
"negative_margin_rate": 0.1395,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9782380353510379,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9215,
|
|
"recall1": 0.9215,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9599166666666666,
|
|
"ndcg3": 0.970373336887502,
|
|
"mean_first_positive_rank": 1.0835,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 12.625836706157774,
|
|
"median_score_margin": 9.696466445922852,
|
|
"negative_margin_rate": 0.0785,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.959310285449028,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9215,
|
|
"recall1": 0.9215,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.95975,
|
|
"ndcg3": 0.9702424071339306,
|
|
"mean_first_positive_rank": 1.0845,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 12.816123366106302,
|
|
"median_score_margin": 9.613193988800049,
|
|
"negative_margin_rate": 0.0785,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.959583455324173,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.951540339499712,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 2000,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.782,
|
|
"recall1": 0.782,
|
|
"recall3": 0.989,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 0.989,
|
|
"hit3": 0.989,
|
|
"mrr": 0.8819583333333334,
|
|
"ndcg3": 0.9077580581071478,
|
|
"mean_first_positive_rank": 1.2795,
|
|
"all_evidence_in_top3": 0.989,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 12.019205482423306,
|
|
"median_score_margin": 8.422776699066162,
|
|
"negative_margin_rate": 0.218,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9750196565389633,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.671,
|
|
"recall1": 0.3355,
|
|
"recall3": 0.912,
|
|
"recall5": 0.99775,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.824,
|
|
"hit3": 0.9855,
|
|
"mrr": 0.8213583333333333,
|
|
"ndcg3": 0.8246288558486577,
|
|
"mean_first_positive_rank": 1.4225,
|
|
"all_evidence_in_top3": 0.8385,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.8385,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 5.122980190102011,
|
|
"median_score_margin": 3.771735429763794,
|
|
"negative_margin_rate": 0.329,
|
|
"hop_confusion": {
|
|
"true_2_pred_0": 2000,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.1813941482268274,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": 0.0,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2000
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.7725,
|
|
"recall1": 0.7725,
|
|
"recall3": 0.8895,
|
|
"recall5": 0.8935,
|
|
"recall8": 0.895,
|
|
"evidence_count_in_top3": 0.8895,
|
|
"hit3": 0.8895,
|
|
"mrr": 0.8329236472757954,
|
|
"ndcg3": 0.8441584402339315,
|
|
"mean_first_positive_rank": 4.2945,
|
|
"all_evidence_in_top3": 0.8895,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 6.7703079060800375,
|
|
"median_score_margin": 4.96909373998642,
|
|
"negative_margin_rate": 0.2275,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9814486785233021,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.942,
|
|
"recall1": 0.942,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.97025,
|
|
"ndcg3": 0.978004741816073,
|
|
"mean_first_positive_rank": 1.0625,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 14.161037405148148,
|
|
"median_score_margin": 10.101282358169556,
|
|
"negative_margin_rate": 0.058,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9777550499141217,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.8845,
|
|
"recall1": 0.8845,
|
|
"recall3": 0.9995,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 0.9995,
|
|
"hit3": 0.9995,
|
|
"mrr": 0.9404583333333334,
|
|
"ndcg3": 0.9557476241250031,
|
|
"mean_first_positive_rank": 1.1265,
|
|
"all_evidence_in_top3": 0.9995,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 10.757258635118603,
|
|
"median_score_margin": 8.949375867843628,
|
|
"negative_margin_rate": 0.1155,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9996015646457672,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.48180815876515987,
|
|
"recall1": 0.48180815876515987,
|
|
"recall3": 0.7425578831312017,
|
|
"recall5": 0.8555678059536935,
|
|
"recall8": 0.9636163175303197,
|
|
"evidence_count_in_top3": 0.7425578831312017,
|
|
"hit3": 0.7425578831312017,
|
|
"mrr": 0.6440939820118917,
|
|
"ndcg3": 0.6360015538470678,
|
|
"mean_first_positive_rank": 2.7210584343991178,
|
|
"all_evidence_in_top3": 0.7425578831312017,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.98125,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.2269306866228252,
|
|
"median_score_margin": -0.02995920181274414,
|
|
"negative_margin_rate": 0.5181918412348401,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 70,
|
|
"true_0_pred_1": 36,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.98632437217459,
|
|
"unknown_mean_need_prob": 0.2532942835329417,
|
|
"thr0.50": {
|
|
"need_precision": 0.9805405405405405,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.990174672489083,
|
|
"specificity_unknown_refusal": 0.660377358490566,
|
|
"abstention_accuracy": 0.98125,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.33962264150943394,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 70,
|
|
"fp": 36,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.3146448188126087,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"V2-512 v3 final": {
|
|
"checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_router_512\\memory_router_v2.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 4741902,
|
|
"checkpoint_bytes": 18973433
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048,
|
|
"checkpoint_bytes": 18973433
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 0.9094866666661497,
|
|
"single_query_latency_ms_p50": 0.8472000008623581,
|
|
"single_query_latency_ms_p95": 1.27029999930528,
|
|
"queries_per_second": 1099.5213417098676,
|
|
"samples": 150,
|
|
"batched_qps_64": 67135.21454097527,
|
|
"batched_latency_ms_64": 0.014895312494900281,
|
|
"batched_qps_256": 168449.866383823,
|
|
"batched_latency_ms_256": 0.005936484376434237
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.8548113333866544,
|
|
"single_query_latency_ms_p50": 0.8628999967186246,
|
|
"single_query_latency_ms_p95": 1.0559000002103858,
|
|
"queries_per_second": 1169.8487852730339,
|
|
"samples": 150,
|
|
"batched_qps_64": 2969.669466452853,
|
|
"batched_latency_ms_64": 0.33673781250627144,
|
|
"batched_qps_256": 3091.357337730013,
|
|
"batched_latency_ms_256": 0.3234824999992725
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.7800044908498933,
|
|
"recall1": 0.7355731447176378,
|
|
"recall3": 0.928146401706523,
|
|
"recall5": 0.9529864151790727,
|
|
"recall8": 0.9671606601549343,
|
|
"evidence_count_in_top3": 1.0318288986190636,
|
|
"hit3": 0.936510609632873,
|
|
"mrr": 0.8612520819605932,
|
|
"ndcg3": 0.8691557053195522,
|
|
"mean_first_positive_rank": 1.9977545750533288,
|
|
"all_evidence_in_top3": 0.9197821937801729,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.849,
|
|
"hop_accuracy": 0.8175182481751825,
|
|
"hop_accuracy_answerable": 0.8877287526664421,
|
|
"hop_under_prediction": 0.11227124733355788,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 53.160489893678886,
|
|
"median_score_margin": 48.923973083496094,
|
|
"negative_margin_rate": 0.21999550915010666,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2106,
|
|
"true_0_pred_1": 2000,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 2000,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.8877236850807078,
|
|
"unknown_mean_need_prob": 0.4528151513680766,
|
|
"thr0.30": {
|
|
"need_precision": 0.8877287526664421,
|
|
"need_recall": 0.8877287526664421,
|
|
"need_f1": 0.8877287526664421,
|
|
"specificity_unknown_refusal": 0.5129079396005846,
|
|
"abstention_accuracy": 0.8175182481751825,
|
|
"known_question_refusal_rate": 0.11227124733355788,
|
|
"unknown_question_read_rate": 0.4870920603994155,
|
|
"counts": {
|
|
"tp": 15814,
|
|
"tn": 2106,
|
|
"fp": 2000,
|
|
"fn": 2000
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.8877287526664421,
|
|
"need_recall": 0.8877287526664421,
|
|
"need_f1": 0.8877287526664421,
|
|
"specificity_unknown_refusal": 0.5129079396005846,
|
|
"abstention_accuracy": 0.8175182481751825,
|
|
"known_question_refusal_rate": 0.11227124733355788,
|
|
"unknown_question_read_rate": 0.4870920603994155,
|
|
"counts": {
|
|
"tp": 15814,
|
|
"tn": 2106,
|
|
"fp": 2000,
|
|
"fn": 2000
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.8877287526664421,
|
|
"need_recall": 0.8877287526664421,
|
|
"need_f1": 0.8877287526664421,
|
|
"specificity_unknown_refusal": 0.5129079396005846,
|
|
"abstention_accuracy": 0.8175182481751825,
|
|
"known_question_refusal_rate": 0.11227124733355788,
|
|
"unknown_question_read_rate": 0.4870920603994155,
|
|
"counts": {
|
|
"tp": 15814,
|
|
"tn": 2106,
|
|
"fp": 2000,
|
|
"fn": 2000
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.9022136010953903,
|
|
"need_recall": 0.8877287526664421,
|
|
"need_f1": 0.8949125686152454,
|
|
"specificity_unknown_refusal": 0.5825621042377009,
|
|
"abstention_accuracy": 0.8305656934306569,
|
|
"known_question_refusal_rate": 0.11227124733355788,
|
|
"unknown_question_read_rate": 0.41743789576229906,
|
|
"counts": {
|
|
"tp": 15814,
|
|
"tn": 2392,
|
|
"fp": 1714,
|
|
"fn": 2000
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 0.9022136010953903,
|
|
"need_recall": 0.8877287526664421,
|
|
"need_f1": 0.8949125686152454,
|
|
"specificity_unknown_refusal": 0.5825621042377009,
|
|
"abstention_accuracy": 0.8305656934306569,
|
|
"known_question_refusal_rate": 0.11227124733355788,
|
|
"unknown_question_read_rate": 0.41743789576229906,
|
|
"counts": {
|
|
"tp": 15814,
|
|
"tn": 2392,
|
|
"fp": 1714,
|
|
"fn": 2000
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 0.9022136010953903,
|
|
"need_recall": 0.8877287526664421,
|
|
"need_f1": 0.8949125686152454,
|
|
"specificity_unknown_refusal": 0.5825621042377009,
|
|
"abstention_accuracy": 0.8305656934306569,
|
|
"known_question_refusal_rate": 0.11227124733355788,
|
|
"unknown_question_read_rate": 0.41743789576229906,
|
|
"counts": {
|
|
"tp": 15814,
|
|
"tn": 2392,
|
|
"fp": 1714,
|
|
"fn": 2000
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 8.474685865469522e-08,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.8178125,
|
|
"recall1": 0.76834375,
|
|
"recall3": 0.9543125,
|
|
"recall5": 0.96865625,
|
|
"recall8": 0.9719375,
|
|
"evidence_count_in_top3": 1.06975,
|
|
"hit3": 0.963625,
|
|
"mrr": 0.8901514232628588,
|
|
"ndcg3": 0.9005039309554683,
|
|
"mean_first_positive_rank": 1.859375,
|
|
"all_evidence_in_top3": 0.945,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.849,
|
|
"hop_accuracy": 0.8,
|
|
"hop_accuracy_answerable": 0.875,
|
|
"hop_under_prediction": 0.125,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 59.239713783336335,
|
|
"median_score_margin": 56.72071409225464,
|
|
"negative_margin_rate": 0.1821875,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000,
|
|
"true_0_pred_1": 2000,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 2000,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.8749969797285945,
|
|
"unknown_mean_need_prob": 0.4644220104073552,
|
|
"thr0.50": {
|
|
"need_precision": 0.875,
|
|
"need_recall": 0.875,
|
|
"need_f1": 0.875,
|
|
"specificity_unknown_refusal": 0.5,
|
|
"abstention_accuracy": 0.8,
|
|
"known_question_refusal_rate": 0.125,
|
|
"unknown_question_read_rate": 0.5,
|
|
"counts": {
|
|
"tp": 14000,
|
|
"tn": 2000,
|
|
"fp": 2000,
|
|
"fn": 2000
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.44639718804920914,
|
|
"recall1": 0.44639718804920914,
|
|
"recall3": 0.6942003514938488,
|
|
"recall5": 0.8096074985354423,
|
|
"recall8": 0.9209138840070299,
|
|
"evidence_count_in_top3": 0.6942003514938488,
|
|
"hit3": 0.6942003514938488,
|
|
"mrr": 0.6050143063165109,
|
|
"ndcg3": 0.5907781161122315,
|
|
"mean_first_positive_rank": 3.256590509666081,
|
|
"all_evidence_in_top3": 0.6942003514938488,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.016779011952367244,
|
|
"median_score_margin": -1.933023452758789,
|
|
"negative_margin_rate": 0.5536028119507909,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9999754249386154,
|
|
"unknown_mean_need_prob": 0.03740391581197314,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.4485981308411215,
|
|
"recall1": 0.4485981308411215,
|
|
"recall3": 0.7476635514018691,
|
|
"recall5": 0.897196261682243,
|
|
"recall8": 0.9906542056074766,
|
|
"evidence_count_in_top3": 0.7476635514018691,
|
|
"hit3": 0.7476635514018691,
|
|
"mrr": 0.6276859341867589,
|
|
"ndcg3": 0.6226036922563473,
|
|
"mean_first_positive_rank": 2.607476635514019,
|
|
"all_evidence_in_top3": 0.7476635514018691,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -7.530951345635351,
|
|
"median_score_margin": -5.711280822753906,
|
|
"negative_margin_rate": 0.5514018691588785,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.03740391581197314,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.8025,
|
|
"recall1": 0.8025,
|
|
"recall3": 0.998,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 0.998,
|
|
"hit3": 0.998,
|
|
"mrr": 0.8965833333333334,
|
|
"ndcg3": 0.9225735229839335,
|
|
"mean_first_positive_rank": 1.2265,
|
|
"all_evidence_in_top3": 0.998,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 60.32892245580256,
|
|
"median_score_margin": 50.02846574783325,
|
|
"negative_margin_rate": 0.1975,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999776878654957,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.898,
|
|
"recall1": 0.898,
|
|
"recall3": 0.9995,
|
|
"recall5": 0.9995,
|
|
"recall8": 0.9995,
|
|
"evidence_count_in_top3": 0.9995,
|
|
"hit3": 0.9995,
|
|
"mrr": 0.9473787878787878,
|
|
"ndcg3": 0.9609264670821456,
|
|
"mean_first_positive_rank": 1.115,
|
|
"all_evidence_in_top3": 0.9995,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 63.02626673298329,
|
|
"median_score_margin": 64.21059799194336,
|
|
"negative_margin_rate": 0.102,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999966541528702,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.906,
|
|
"recall1": 0.906,
|
|
"recall3": 0.9995,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 0.9995,
|
|
"hit3": 0.9995,
|
|
"mrr": 0.9512916666666666,
|
|
"ndcg3": 0.9637480993000024,
|
|
"mean_first_positive_rank": 1.1045,
|
|
"all_evidence_in_top3": 0.9995,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 63.26017077337205,
|
|
"median_score_margin": 64.34984970092773,
|
|
"negative_margin_rate": 0.094,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999967547059059,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.9287911344170571,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 2000,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.6965,
|
|
"recall1": 0.6965,
|
|
"recall3": 0.8465,
|
|
"recall5": 0.8605,
|
|
"recall8": 0.8695,
|
|
"evidence_count_in_top3": 0.8465,
|
|
"hit3": 0.8465,
|
|
"mrr": 0.776365632860676,
|
|
"ndcg3": 0.7873425001821464,
|
|
"mean_first_positive_rank": 4.702,
|
|
"all_evidence_in_top3": 0.8465,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 59.559096137337384,
|
|
"median_score_margin": 77.47207641601562,
|
|
"negative_margin_rate": 0.3035,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.7915,
|
|
"recall1": 0.39575,
|
|
"recall3": 0.9235,
|
|
"recall5": 0.99575,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.847,
|
|
"hit3": 0.998,
|
|
"mrr": 0.891,
|
|
"ndcg3": 0.8664546282365805,
|
|
"mean_first_positive_rank": 1.238,
|
|
"all_evidence_in_top3": 0.849,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.849,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 34.953358390808106,
|
|
"median_score_margin": 41.703819274902344,
|
|
"negative_margin_rate": 0.2085,
|
|
"hop_confusion": {
|
|
"true_2_pred_0": 2000,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 9.34347713071304e-06,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": 0.0,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2000
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.625,
|
|
"recall1": 0.625,
|
|
"recall3": 0.8685,
|
|
"recall5": 0.8935,
|
|
"recall8": 0.9065,
|
|
"evidence_count_in_top3": 0.8685,
|
|
"hit3": 0.8685,
|
|
"mrr": 0.7501752986967395,
|
|
"ndcg3": 0.7711683990410768,
|
|
"mean_first_positive_rank": 3.293,
|
|
"all_evidence_in_top3": 0.8685,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 22.522678687380626,
|
|
"median_score_margin": 15.791908264160156,
|
|
"negative_margin_rate": 0.375,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.999996542096138,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.92,
|
|
"recall1": 0.92,
|
|
"recall3": 0.999,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 0.999,
|
|
"hit3": 0.999,
|
|
"mrr": 0.9584166666666666,
|
|
"ndcg3": 0.9687960125035735,
|
|
"mean_first_positive_rank": 1.09,
|
|
"all_evidence_in_top3": 0.999,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 101.63241866916046,
|
|
"median_score_margin": 87.8995840549469,
|
|
"negative_margin_rate": 0.08,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999988555312157,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.903,
|
|
"recall1": 0.903,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.95,
|
|
"ndcg3": 0.9630218183142882,
|
|
"mean_first_positive_rank": 1.106,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 68.63479841984622,
|
|
"median_score_margin": 62.0030552148819,
|
|
"negative_margin_rate": 0.097,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.44652701212789414,
|
|
"recall1": 0.44652701212789414,
|
|
"recall3": 0.697353914002205,
|
|
"recall5": 0.814773980154355,
|
|
"recall8": 0.9250275633958104,
|
|
"evidence_count_in_top3": 0.697353914002205,
|
|
"hit3": 0.697353914002205,
|
|
"mrr": 0.606351607409188,
|
|
"ndcg3": 0.5926553689498392,
|
|
"mean_first_positive_rank": 3.2183020948180814,
|
|
"all_evidence_in_top3": 0.697353914002205,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.46000747926442864,
|
|
"median_score_margin": -2.0159082412719727,
|
|
"negative_margin_rate": 0.5534729878721059,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9999768745150035,
|
|
"unknown_mean_need_prob": 0.01482047064058326,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 5.2886397653310265e-05,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"V2-512 v6 best": {
|
|
"checkpoint": "checkpoints\\router_v6_v2_512\\router_best.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": 2000,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 4741902,
|
|
"checkpoint_bytes": 56922573
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048,
|
|
"checkpoint_bytes": 56922573
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 0.8750353333986519,
|
|
"single_query_latency_ms_p50": 0.8467999978165608,
|
|
"single_query_latency_ms_p95": 1.2185999985376839,
|
|
"queries_per_second": 1142.8109949754637,
|
|
"samples": 150,
|
|
"batched_qps_64": 73617.37369428946,
|
|
"batched_latency_ms_64": 0.013583750001089356,
|
|
"batched_qps_256": 161941.27100652116,
|
|
"batched_latency_ms_256": 0.006175078124215361
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.921385333276703,
|
|
"single_query_latency_ms_p50": 0.8951000017987099,
|
|
"single_query_latency_ms_p95": 1.1334999981045257,
|
|
"queries_per_second": 1085.3222467126986,
|
|
"samples": 150,
|
|
"batched_qps_64": 2924.674104534912,
|
|
"batched_latency_ms_64": 0.3419184374934048,
|
|
"batched_qps_256": 3073.5827721960004,
|
|
"batched_latency_ms_256": 0.32535320312376825
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.950488379925901,
|
|
"recall1": 0.8971034018187942,
|
|
"recall3": 0.9884079937128102,
|
|
"recall5": 0.9934882676546536,
|
|
"recall8": 0.9961266419669923,
|
|
"evidence_count_in_top3": 1.0986302907825305,
|
|
"hit3": 0.9904569439766476,
|
|
"mrr": 0.9711571604054701,
|
|
"ndcg3": 0.9723689530227545,
|
|
"mean_first_positive_rank": 1.1156393847535646,
|
|
"all_evidence_in_top3": 0.9863590434489727,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9635,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 12.36926158705623,
|
|
"median_score_margin": 12.002254128456116,
|
|
"negative_margin_rate": 0.049511620074099025,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9988408471058552,
|
|
"unknown_mean_need_prob": 0.006478862109410833,
|
|
"thr0.30": {
|
|
"need_precision": 0.999326825984517,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9996632996632996,
|
|
"specificity_unknown_refusal": 0.9970774476376035,
|
|
"abstention_accuracy": 0.9994525547445255,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0029225523623964927,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4094,
|
|
"fp": 12,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.999326825984517,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9996632996632996,
|
|
"specificity_unknown_refusal": 0.9970774476376035,
|
|
"abstention_accuracy": 0.9994525547445255,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0029225523623964927,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4094,
|
|
"fp": 12,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.9995511165974638,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.999775507913346,
|
|
"specificity_unknown_refusal": 0.9980516317584024,
|
|
"abstention_accuracy": 0.9996350364963503,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.001948368241597662,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4098,
|
|
"fp": 8,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.9998877413560844,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9999438675273646,
|
|
"specificity_unknown_refusal": 0.9995129079396006,
|
|
"abstention_accuracy": 0.9999087591240876,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0004870920603994155,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4104,
|
|
"fp": 2,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.0904978112084791,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.97,
|
|
"recall1": 0.9105625,
|
|
"recall3": 0.99771875,
|
|
"recall5": 0.9999375,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.1204375,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9849166666666667,
|
|
"ndcg3": 0.9857856193167982,
|
|
"mean_first_positive_rank": 1.0305,
|
|
"all_evidence_in_top3": 0.9954375,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9635,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 13.542529965203517,
|
|
"median_score_margin": 12.727105312049389,
|
|
"negative_margin_rate": 0.03,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9990760059095919,
|
|
"unknown_mean_need_prob": 0.0025661538663334796,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.8125366139425894,
|
|
"recall1": 0.8125366139425894,
|
|
"recall3": 0.9244288224956063,
|
|
"recall5": 0.9502050380785003,
|
|
"recall8": 0.971294669009959,
|
|
"evidence_count_in_top3": 0.9244288224956063,
|
|
"hit3": 0.9244288224956063,
|
|
"mrr": 0.8742941912651278,
|
|
"ndcg3": 0.878837443896981,
|
|
"mean_first_positive_rank": 1.7176332747510252,
|
|
"all_evidence_in_top3": 0.9244288224956063,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 2.2332465364638656,
|
|
"median_score_margin": 2.495278835296631,
|
|
"negative_margin_rate": 0.18746338605741067,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.996912036259131,
|
|
"unknown_mean_need_prob": 0.2510888675848643,
|
|
"thr0.50": {
|
|
"need_precision": 0.9976621858562245,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9988297249853716,
|
|
"specificity_unknown_refusal": 0.8095238095238095,
|
|
"abstention_accuracy": 0.9976851851851852,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.19047619047619047,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 17,
|
|
"fp": 4,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.2336448598130841,
|
|
"recall1": 0.2336448598130841,
|
|
"recall3": 0.616822429906542,
|
|
"recall5": 0.719626168224299,
|
|
"recall8": 0.8130841121495327,
|
|
"evidence_count_in_top3": 0.616822429906542,
|
|
"hit3": 0.616822429906542,
|
|
"mrr": 0.4589420963252739,
|
|
"ndcg3": 0.4582719938918631,
|
|
"mean_first_positive_rank": 4.242990654205608,
|
|
"all_evidence_in_top3": 0.616822429906542,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -1.3701436372003823,
|
|
"median_score_margin": -0.6190404891967773,
|
|
"negative_margin_rate": 0.7663551401869159,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 0.9944477560364198,
|
|
"unknown_mean_need_prob": 0.2510888675848643,
|
|
"thr0.50": {
|
|
"need_precision": 0.963963963963964,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9816513761467891,
|
|
"specificity_unknown_refusal": 0.8095238095238095,
|
|
"abstention_accuracy": 0.96875,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.19047619047619047,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 17,
|
|
"fp": 4,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9905,
|
|
"recall1": 0.9905,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99525,
|
|
"ndcg3": 0.9964938326589288,
|
|
"mean_first_positive_rank": 1.0095,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 16.784344019554556,
|
|
"median_score_margin": 14.332054734230042,
|
|
"negative_margin_rate": 0.0095,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9984212898612023,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.974,
|
|
"recall1": 0.974,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.987,
|
|
"ndcg3": 0.9904041735928579,
|
|
"mean_first_positive_rank": 1.026,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 14.32004562981613,
|
|
"median_score_margin": 12.83707270026207,
|
|
"negative_margin_rate": 0.026,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9998619016408921,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.969,
|
|
"recall1": 0.969,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9844166666666666,
|
|
"ndcg3": 0.9884933574839294,
|
|
"mean_first_positive_rank": 1.0315,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 14.515158921549562,
|
|
"median_score_margin": 12.90774929523468,
|
|
"negative_margin_rate": 0.031,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9998633873462677,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.004985218498797622,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.973,
|
|
"recall1": 0.973,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9864166666666666,
|
|
"ndcg3": 0.9899696384696437,
|
|
"mean_first_positive_rank": 1.0275,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 11.82059130709432,
|
|
"median_score_margin": 12.446841090917587,
|
|
"negative_margin_rate": 0.027,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9998804149031639,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.951,
|
|
"recall1": 0.4755,
|
|
"recall3": 0.98175,
|
|
"recall5": 0.9995,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.9635,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9753333333333334,
|
|
"ndcg3": 0.9571701608629515,
|
|
"mean_first_positive_rank": 1.05,
|
|
"all_evidence_in_top3": 0.9635,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9635,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 10.0001081677787,
|
|
"median_score_margin": 11.238629460334778,
|
|
"negative_margin_rate": 0.049,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9978087923228741,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9755,
|
|
"recall1": 0.9755,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98775,
|
|
"ndcg3": 0.9909577789625007,
|
|
"mean_first_positive_rank": 1.0245,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 14.530858453191817,
|
|
"median_score_margin": 12.788989305496216,
|
|
"negative_margin_rate": 0.0245,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9972889910042286,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9745,
|
|
"recall1": 0.9745,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9871666666666666,
|
|
"ndcg3": 0.9905232438392865,
|
|
"mean_first_positive_rank": 1.026,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 13.56638970718137,
|
|
"median_score_margin": 13.154640913009644,
|
|
"negative_margin_rate": 0.0255,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9995595035254955,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9525,
|
|
"recall1": 0.9525,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.976,
|
|
"ndcg3": 0.982272768664287,
|
|
"mean_first_positive_rank": 1.049,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 12.802743515461684,
|
|
"median_score_margin": 12.365995168685913,
|
|
"negative_margin_rate": 0.0475,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999237666726113,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.7783902976846747,
|
|
"recall1": 0.7783902976846747,
|
|
"recall3": 0.906284454244763,
|
|
"recall5": 0.9366041896361632,
|
|
"recall8": 0.961962513781698,
|
|
"evidence_count_in_top3": 0.906284454244763,
|
|
"hit3": 0.906284454244763,
|
|
"mrr": 0.8497943708910569,
|
|
"ndcg3": 0.8540301102968997,
|
|
"mean_first_positive_rank": 1.866593164277839,
|
|
"all_evidence_in_top3": 0.906284454244763,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 2.0206981634858754,
|
|
"median_score_margin": 2.3058557510375977,
|
|
"negative_margin_rate": 0.22160970231532526,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9967666790464352,
|
|
"unknown_mean_need_prob": 0.1541282297727072,
|
|
"thr0.50": {
|
|
"need_precision": 0.995609220636663,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9977997799779977,
|
|
"specificity_unknown_refusal": 0.9245283018867925,
|
|
"abstention_accuracy": 0.9958333333333333,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.07547169811320754,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 98,
|
|
"fp": 8,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 0.00014708923386933748,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"V2-512 v6 final": {
|
|
"checkpoint": "checkpoints\\router_v6_v2_512\\memory_router_v2.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 4741902,
|
|
"checkpoint_bytes": 18973433
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048,
|
|
"checkpoint_bytes": 18973433
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 0.8725079999809774,
|
|
"single_query_latency_ms_p50": 0.850600001285784,
|
|
"single_query_latency_ms_p95": 1.0982999992847908,
|
|
"queries_per_second": 1146.121296334019,
|
|
"samples": 150,
|
|
"batched_qps_64": 61636.8434007473,
|
|
"batched_latency_ms_64": 0.016224062505898473,
|
|
"batched_qps_256": 164167.81025689258,
|
|
"batched_latency_ms_256": 0.006091328125990003
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.8932059997459874,
|
|
"single_query_latency_ms_p50": 0.876300000527408,
|
|
"single_query_latency_ms_p95": 1.1406000012357254,
|
|
"queries_per_second": 1119.5625648331775,
|
|
"samples": 150,
|
|
"batched_qps_64": 2943.6168587607826,
|
|
"batched_latency_ms_64": 0.3397181250079484,
|
|
"batched_qps_256": 3143.215455374193,
|
|
"batched_latency_ms_256": 0.3181455468762806
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.941843493881217,
|
|
"recall1": 0.8864657011339396,
|
|
"recall3": 0.9666273717300999,
|
|
"recall5": 0.9764230380599529,
|
|
"recall8": 0.984506567867969,
|
|
"evidence_count_in_top3": 1.07668126192882,
|
|
"hit3": 0.9688447288649377,
|
|
"mrr": 0.958724954839147,
|
|
"ndcg3": 0.9548399733910118,
|
|
"mean_first_positive_rank": 1.2825867295385651,
|
|
"all_evidence_in_top3": 0.9644100145952621,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9605,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 73.24820388849786,
|
|
"median_score_margin": 76.59281730651855,
|
|
"negative_margin_rate": 0.05815650611878298,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999984274064,
|
|
"unknown_mean_need_prob": 4.522551017238816e-06,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 4.432710963581804e-06,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.98375,
|
|
"recall1": 0.92209375,
|
|
"recall3": 0.99753125,
|
|
"recall5": 0.9999375,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.1200625,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99184375,
|
|
"ndcg3": 0.9902417180168516,
|
|
"mean_first_positive_rank": 1.0164375,
|
|
"all_evidence_in_top3": 0.9950625,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9605,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 81.85253369444608,
|
|
"median_score_margin": 83.23931503295898,
|
|
"negative_margin_rate": 0.01625,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999999701977,
|
|
"unknown_mean_need_prob": 6.058075942262249e-11,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.5770357352079672,
|
|
"recall1": 0.5770357352079672,
|
|
"recall3": 0.6859988283538372,
|
|
"recall5": 0.7562975981253661,
|
|
"recall8": 0.8388986526069128,
|
|
"evidence_count_in_top3": 0.6859988283538372,
|
|
"hit3": 0.6859988283538372,
|
|
"mrr": 0.6663638286240626,
|
|
"ndcg3": 0.6401845706816489,
|
|
"mean_first_positive_rank": 3.719976567076743,
|
|
"all_evidence_in_top3": 0.6859988283538372,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -2.648404171182946,
|
|
"median_score_margin": 5.544002532958984,
|
|
"negative_margin_rate": 0.4229642647920328,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9999999838679872,
|
|
"unknown_mean_need_prob": 0.00043537282504942034,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.4953271028037383,
|
|
"recall1": 0.4953271028037383,
|
|
"recall3": 0.822429906542056,
|
|
"recall5": 0.9719626168224299,
|
|
"recall8": 0.9906542056074766,
|
|
"evidence_count_in_top3": 0.822429906542056,
|
|
"hit3": 0.822429906542056,
|
|
"mrr": 0.6704980377877574,
|
|
"ndcg3": 0.680904070694264,
|
|
"mean_first_positive_rank": 2.196261682242991,
|
|
"all_evidence_in_top3": 0.822429906542056,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -2.581393656329574,
|
|
"median_score_margin": -0.056890010833740234,
|
|
"negative_margin_rate": 0.5046728971962616,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.00043537282504942034,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9935,
|
|
"recall1": 0.9935,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99675,
|
|
"ndcg3": 0.9976010433982145,
|
|
"mean_first_positive_rank": 1.0065,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 64.64610423742235,
|
|
"median_score_margin": 60.52951920032501,
|
|
"negative_margin_rate": 0.0065,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.98,
|
|
"recall1": 0.98,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99,
|
|
"ndcg3": 0.9926185950714291,
|
|
"mean_first_positive_rank": 1.02,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 88.94486605204642,
|
|
"median_score_margin": 95.25923618674278,
|
|
"negative_margin_rate": 0.02,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9805,
|
|
"recall1": 0.9805,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9901666666666666,
|
|
"ndcg3": 0.9927376653178577,
|
|
"mean_first_positive_rank": 1.02,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 90.02319332621246,
|
|
"median_score_margin": 96.11362898349762,
|
|
"negative_margin_rate": 0.0195,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 1.2115745555928435e-10,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.977,
|
|
"recall1": 0.977,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9884166666666666,
|
|
"ndcg3": 0.9914459194553578,
|
|
"mean_first_positive_rank": 1.0235,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 78.39938361007721,
|
|
"median_score_margin": 81.58803009986877,
|
|
"negative_margin_rate": 0.023,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9865,
|
|
"recall1": 0.49325,
|
|
"recall3": 0.98025,
|
|
"recall5": 0.9995,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.9605,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99325,
|
|
"ndcg3": 0.9651268224740949,
|
|
"mean_first_positive_rank": 1.0135,
|
|
"all_evidence_in_top3": 0.9605,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.9605,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 104.36254599650204,
|
|
"median_score_margin": 118.10550880432129,
|
|
"negative_margin_rate": 0.0135,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999997615814,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9955,
|
|
"recall1": 0.9955,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99775,
|
|
"ndcg3": 0.9983391838910716,
|
|
"mean_first_positive_rank": 1.0045,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 57.61693591082841,
|
|
"median_score_margin": 57.403422355651855,
|
|
"negative_margin_rate": 0.0045,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9825,
|
|
"recall1": 0.9825,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99125,
|
|
"ndcg3": 0.9935412706875005,
|
|
"mean_first_positive_rank": 1.0175,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 84.6615379076004,
|
|
"median_score_margin": 90.02162456512451,
|
|
"negative_margin_rate": 0.0175,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9745,
|
|
"recall1": 0.9745,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9871666666666666,
|
|
"ndcg3": 0.9905232438392865,
|
|
"mean_first_positive_rank": 1.026,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 86.1657025148794,
|
|
"median_score_margin": 91.7922420501709,
|
|
"negative_margin_rate": 0.0255,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.5722160970231532,
|
|
"recall1": 0.5722160970231532,
|
|
"recall3": 0.6940463065049615,
|
|
"recall5": 0.7690187431091511,
|
|
"recall8": 0.8478500551267916,
|
|
"evidence_count_in_top3": 0.6940463065049615,
|
|
"hit3": 0.6940463065049615,
|
|
"mrr": 0.666607687709242,
|
|
"ndcg3": 0.6425864375511913,
|
|
"mean_first_positive_rank": 3.630099228224917,
|
|
"all_evidence_in_top3": 0.6940463065049615,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -2.644451511266016,
|
|
"median_score_margin": 4.475192725658417,
|
|
"negative_margin_rate": 0.42778390297684676,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9999999848195447,
|
|
"unknown_mean_need_prob": 0.00017518256748815932,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 4.063285960620704e-15,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"XL-512 v6 best": {
|
|
"checkpoint": "checkpoints\\router_v6_xl512\\router_best.pt",
|
|
"router": {
|
|
"kind": "router_xl",
|
|
"step": 36000,
|
|
"arch_config": {
|
|
"arch": "router_xl",
|
|
"arch_version": 1,
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3,
|
|
"encoder_layers": 2,
|
|
"encoder_hidden": 512,
|
|
"pair_blocks": 1,
|
|
"pair_hidden": 512,
|
|
"pair_expansion": 2,
|
|
"pair_dropout": 0.05,
|
|
"use_interaction": true,
|
|
"policy_layers": 2,
|
|
"policy_hidden": 512,
|
|
"policy_dropout": 0.0,
|
|
"learnable_cosine_scale": true
|
|
},
|
|
"parameters": 7898127,
|
|
"checkpoint_bytes": 99014505
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048,
|
|
"checkpoint_bytes": 99014505
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 1.2161159999474573,
|
|
"single_query_latency_ms_p50": 1.1961999989580363,
|
|
"single_query_latency_ms_p95": 1.5090000015334226,
|
|
"queries_per_second": 822.2899789520122,
|
|
"samples": 150,
|
|
"batched_qps_64": 57386.75086015149,
|
|
"batched_latency_ms_64": 0.017425624992029043,
|
|
"batched_qps_256": 96746.88596045674,
|
|
"batched_latency_ms_256": 0.010336249999909342
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 1.6496533333702246,
|
|
"single_query_latency_ms_p50": 1.6746999972383492,
|
|
"single_query_latency_ms_p95": 2.007999999477761,
|
|
"queries_per_second": 606.1879667511782,
|
|
"samples": 150,
|
|
"batched_qps_64": 1660.2789164532574,
|
|
"batched_latency_ms_64": 0.6023084375101462,
|
|
"batched_qps_256": 1699.1487264831305,
|
|
"batched_latency_ms_256": 0.5885300000016969
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.957842146626249,
|
|
"recall1": 0.9024082182553048,
|
|
"recall3": 0.9758897496351184,
|
|
"recall5": 0.9834399910183003,
|
|
"recall8": 0.9905411474121477,
|
|
"evidence_count_in_top3": 1.0808914337038285,
|
|
"hit3": 0.9831593128999663,
|
|
"mrr": 0.9724377931581535,
|
|
"ndcg3": 0.9664212737428025,
|
|
"mean_first_positive_rank": 1.1337150555742674,
|
|
"all_evidence_in_top3": 0.9686201863702706,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.8705,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 45.56406442258217,
|
|
"median_score_margin": 47.89992094039917,
|
|
"negative_margin_rate": 0.042157853373750984,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999977515257,
|
|
"unknown_mean_need_prob": 9.165001448203187e-06,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 1.130629900134017e-05,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.98375,
|
|
"recall1": 0.92203125,
|
|
"recall3": 0.99190625,
|
|
"recall5": 0.994875,
|
|
"recall8": 0.99640625,
|
|
"evidence_count_in_top3": 1.1088125,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99184375,
|
|
"ndcg3": 0.9863265131728122,
|
|
"mean_first_positive_rank": 1.0164375,
|
|
"all_evidence_in_top3": 0.9838125,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.8705,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 48.28189839535998,
|
|
"median_score_margin": 49.04537320137024,
|
|
"negative_margin_rate": 0.01625,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999999552965,
|
|
"unknown_mean_need_prob": 2.862315262235959e-10,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.7457527826596367,
|
|
"recall1": 0.7457527826596367,
|
|
"recall3": 0.8347978910369068,
|
|
"recall5": 0.8763913298183948,
|
|
"recall8": 0.9355594610427651,
|
|
"evidence_count_in_top3": 0.8347978910369068,
|
|
"hit3": 0.8347978910369068,
|
|
"mrr": 0.8100220298155272,
|
|
"ndcg3": 0.7979455040170743,
|
|
"mean_first_positive_rank": 2.16813122437024,
|
|
"all_evidence_in_top3": 0.8347978910369068,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 22.8850553075215,
|
|
"median_score_margin": 27.899999618530273,
|
|
"negative_margin_rate": 0.2542472173403632,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9999999769542673,
|
|
"unknown_mean_need_prob": 0.0008787320924793242,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.4672897196261682,
|
|
"recall1": 0.4672897196261682,
|
|
"recall3": 0.8317757009345794,
|
|
"recall5": 0.9813084112149533,
|
|
"recall8": 0.9906542056074766,
|
|
"evidence_count_in_top3": 0.8317757009345794,
|
|
"hit3": 0.8317757009345794,
|
|
"mrr": 0.6616751628433871,
|
|
"ndcg3": 0.6776764890854534,
|
|
"mean_first_positive_rank": 2.1682242990654204,
|
|
"all_evidence_in_top3": 0.8317757009345794,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.9633634409336286,
|
|
"median_score_margin": -1.062392234802246,
|
|
"negative_margin_rate": 0.5327102803738317,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0008787320924793242,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.992,
|
|
"recall1": 0.992,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.996,
|
|
"ndcg3": 0.9970474380285717,
|
|
"mean_first_positive_rank": 1.008,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 67.69295654568822,
|
|
"median_score_margin": 69.3216142654419,
|
|
"negative_margin_rate": 0.008,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9795,
|
|
"recall1": 0.9795,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.98975,
|
|
"ndcg3": 0.9924340599482149,
|
|
"mean_first_positive_rank": 1.0205,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 39.98083441020921,
|
|
"median_score_margin": 44.252100229263306,
|
|
"negative_margin_rate": 0.0205,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9805,
|
|
"recall1": 0.9805,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99025,
|
|
"ndcg3": 0.9928031301946435,
|
|
"mean_first_positive_rank": 1.0195,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 40.70788796337694,
|
|
"median_score_margin": 44.594672441482544,
|
|
"negative_margin_rate": 0.0195,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 5.724454182644648e-10,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9815,
|
|
"recall1": 0.9815,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9906666666666666,
|
|
"ndcg3": 0.9931067355642863,
|
|
"mean_first_positive_rank": 1.019,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 37.93500039816182,
|
|
"median_score_margin": 40.258564949035645,
|
|
"negative_margin_rate": 0.0185,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9875,
|
|
"recall1": 0.49375,
|
|
"recall3": 0.93525,
|
|
"recall5": 0.959,
|
|
"recall8": 0.97125,
|
|
"evidence_count_in_top3": 1.8705,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9936666666666666,
|
|
"ndcg3": 0.9341087890914226,
|
|
"mean_first_positive_rank": 1.013,
|
|
"all_evidence_in_top3": 0.8705,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.8705,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 47.13378771021217,
|
|
"median_score_margin": 51.072168588638306,
|
|
"negative_margin_rate": 0.0125,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999996423722,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9965,
|
|
"recall1": 0.9965,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99825,
|
|
"ndcg3": 0.9987082541375002,
|
|
"mean_first_positive_rank": 1.0035,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 61.5167154010348,
|
|
"median_score_margin": 62.94392251968384,
|
|
"negative_margin_rate": 0.0035,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9815,
|
|
"recall1": 0.9815,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99075,
|
|
"ndcg3": 0.9931722004410719,
|
|
"mean_first_positive_rank": 1.0185,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 38.85421975574363,
|
|
"median_score_margin": 43.87380111217499,
|
|
"negative_margin_rate": 0.0185,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.971,
|
|
"recall1": 0.971,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9854166666666666,
|
|
"ndcg3": 0.9892314979767864,
|
|
"mean_first_positive_rank": 1.0295,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 52.43378497845307,
|
|
"median_score_margin": 59.86355948448181,
|
|
"negative_margin_rate": 0.029,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.7293274531422271,
|
|
"recall1": 0.7293274531422271,
|
|
"recall3": 0.834619625137817,
|
|
"recall5": 0.88257993384785,
|
|
"recall8": 0.9388092613009923,
|
|
"evidence_count_in_top3": 0.834619625137817,
|
|
"hit3": 0.834619625137817,
|
|
"mrr": 0.8012716909147449,
|
|
"ndcg3": 0.7908513559477891,
|
|
"mean_first_positive_rank": 2.1681367144432193,
|
|
"all_evidence_in_top3": 0.834619625137817,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 21.591989690253087,
|
|
"median_score_margin": 24.97784423828125,
|
|
"negative_margin_rate": 0.27067254685777287,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9999999783136353,
|
|
"unknown_mean_need_prob": 0.0003550033115114848,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 1.7634182726834057e-14,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"XL-512 v6 final": {
|
|
"checkpoint": "checkpoints\\router_v6_xl512\\memory_router_xl.pt",
|
|
"router": {
|
|
"kind": "router_xl",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_xl",
|
|
"arch_version": 1,
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3,
|
|
"encoder_layers": 2,
|
|
"encoder_hidden": 512,
|
|
"pair_blocks": 1,
|
|
"pair_hidden": 512,
|
|
"pair_expansion": 2,
|
|
"pair_dropout": 0.05,
|
|
"use_interaction": true,
|
|
"policy_layers": 2,
|
|
"policy_hidden": 512,
|
|
"policy_dropout": 0.0,
|
|
"learnable_cosine_scale": true
|
|
},
|
|
"parameters": 7898127,
|
|
"checkpoint_bytes": 35802155
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048,
|
|
"checkpoint_bytes": 35802155
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 1.2505186665900208,
|
|
"single_query_latency_ms_p50": 1.2239999996381812,
|
|
"single_query_latency_ms_p95": 1.6236000010394491,
|
|
"queries_per_second": 799.6681910609555,
|
|
"samples": 150,
|
|
"batched_qps_64": 53291.58828255585,
|
|
"batched_latency_ms_64": 0.018764687490602228,
|
|
"batched_qps_256": 94345.88082639035,
|
|
"batched_latency_ms_256": 0.010599296876989683
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 1.717839333335481,
|
|
"single_query_latency_ms_p50": 1.7062999977497384,
|
|
"single_query_latency_ms_p95": 2.0226999986334704,
|
|
"queries_per_second": 582.1266171955253,
|
|
"samples": 150,
|
|
"batched_qps_64": 1690.8261059433491,
|
|
"batched_latency_ms_64": 0.5914268749961593,
|
|
"batched_qps_256": 1707.8817676137278,
|
|
"batched_latency_ms_256": 0.585520625000413
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 21920,
|
|
"answerable_episodes": 17814,
|
|
"unknown_episodes": 4106,
|
|
"top1": 0.9502077018075671,
|
|
"recall1": 0.8948299090602897,
|
|
"recall3": 0.965336252385764,
|
|
"recall5": 0.9760862243179522,
|
|
"recall8": 0.9854608734703043,
|
|
"evidence_count_in_top3": 1.0657348153137982,
|
|
"hit3": 0.9772089367912877,
|
|
"mrr": 0.9664213223997663,
|
|
"ndcg3": 0.9560053718886551,
|
|
"mean_first_positive_rank": 1.1716066015493432,
|
|
"all_evidence_in_top3": 0.9534635679802402,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.7885,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 215.75331382084113,
|
|
"median_score_margin": 239.68688774108887,
|
|
"negative_margin_rate": 0.049792298192432916,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 15814,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999979790499,
|
|
"unknown_mean_need_prob": 4.1244088529706584e-06,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 17814,
|
|
"tn": 4106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 4.264552800687521e-06,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20000,
|
|
"answerable_episodes": 16000,
|
|
"unknown_episodes": 4000,
|
|
"top1": 0.984,
|
|
"recall1": 0.92234375,
|
|
"recall3": 0.98678125,
|
|
"recall5": 0.990875,
|
|
"recall8": 0.9933125,
|
|
"evidence_count_in_top3": 1.0985625,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99196875,
|
|
"ndcg3": 0.9823026449115756,
|
|
"mean_first_positive_rank": 1.0161875,
|
|
"all_evidence_in_top3": 0.9735625,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.7885,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 235.8910635419197,
|
|
"median_score_margin": 250.7837634086609,
|
|
"negative_margin_rate": 0.016,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 4000,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 14000,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 0,
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999999627471,
|
|
"unknown_mean_need_prob": 7.011441629130955e-11,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16000,
|
|
"tn": 4000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.6690099589923842,
|
|
"recall1": 0.6690099589923842,
|
|
"recall3": 0.7709431751611013,
|
|
"recall5": 0.8388986526069128,
|
|
"recall8": 0.9121265377855887,
|
|
"evidence_count_in_top3": 0.7709431751611013,
|
|
"hit3": 0.7709431751611013,
|
|
"mrr": 0.7488416468644941,
|
|
"ndcg3": 0.7281836459473614,
|
|
"mean_first_positive_rank": 2.558289396602226,
|
|
"all_evidence_in_top3": 0.7709431751611013,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 40.85260479970152,
|
|
"median_score_margin": 62.14678978919983,
|
|
"negative_margin_rate": 0.3309900410076157,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9999999792588407,
|
|
"unknown_mean_need_prob": 0.00039670502174734185,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.38317757009345793,
|
|
"recall1": 0.38317757009345793,
|
|
"recall3": 0.8598130841121495,
|
|
"recall5": 0.9532710280373832,
|
|
"recall8": 0.9813084112149533,
|
|
"evidence_count_in_top3": 0.8598130841121495,
|
|
"hit3": 0.8598130841121495,
|
|
"mrr": 0.617352766651832,
|
|
"ndcg3": 0.6582046038050816,
|
|
"mean_first_positive_rank": 2.289719626168224,
|
|
"all_evidence_in_top3": 0.8598130841121495,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -5.260566909736562,
|
|
"median_score_margin": -7.525552272796631,
|
|
"negative_margin_rate": 0.616822429906542,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.00039670502174734185,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9925,
|
|
"recall1": 0.9925,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99625,
|
|
"ndcg3": 0.9972319731517859,
|
|
"mean_first_positive_rank": 1.0075,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 272.5490972074904,
|
|
"median_score_margin": 288.63626432418823,
|
|
"negative_margin_rate": 0.0075,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.98,
|
|
"recall1": 0.98,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99,
|
|
"ndcg3": 0.9926185950714291,
|
|
"mean_first_positive_rank": 1.02,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 215.3791910160221,
|
|
"median_score_margin": 239.17540168762207,
|
|
"negative_margin_rate": 0.02,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.982,
|
|
"recall1": 0.982,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9909166666666667,
|
|
"ndcg3": 0.9932912706875006,
|
|
"mean_first_positive_rank": 1.0185,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 219.1182684260085,
|
|
"median_score_margin": 239.48388767242432,
|
|
"negative_margin_rate": 0.018,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 1.4022628460683615e-10,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.983,
|
|
"recall1": 0.983,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9914166666666666,
|
|
"ndcg3": 0.9936603409339291,
|
|
"mean_first_positive_rank": 1.0175,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 211.44927242165804,
|
|
"median_score_margin": 233.2896122932434,
|
|
"negative_margin_rate": 0.017,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9865,
|
|
"recall1": 0.49325,
|
|
"recall3": 0.89425,
|
|
"recall5": 0.927,
|
|
"recall8": 0.9465,
|
|
"evidence_count_in_top3": 1.7885,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99325,
|
|
"ndcg3": 0.9008760971390303,
|
|
"mean_first_positive_rank": 1.0135,
|
|
"all_evidence_in_top3": 0.7885,
|
|
"multi_positive_episodes": 2000,
|
|
"all_evidence_in_top3_multi": 0.7885,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 274.43398497897016,
|
|
"median_score_margin": 301.56740951538086,
|
|
"negative_margin_rate": 0.0135,
|
|
"hop_confusion": {
|
|
"true_2_pred_2": 2000
|
|
},
|
|
"known_mean_need_prob": 0.9999999997019767,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.995,
|
|
"recall1": 0.995,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9975,
|
|
"ndcg3": 0.9981546487678573,
|
|
"mean_first_positive_rank": 1.005,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 223.85322645658067,
|
|
"median_score_margin": 222.2765736579895,
|
|
"negative_margin_rate": 0.005,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9805,
|
|
"recall1": 0.9805,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.99025,
|
|
"ndcg3": 0.9928031301946435,
|
|
"mean_first_positive_rank": 1.0195,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 225.64811652239038,
|
|
"median_score_margin": 241.96742022037506,
|
|
"negative_margin_rate": 0.0195,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 2000,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.9725,
|
|
"recall1": 0.9725,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"evidence_count_in_top3": 1.0,
|
|
"hit3": 1.0,
|
|
"mrr": 0.9861666666666666,
|
|
"ndcg3": 0.9897851033464293,
|
|
"mean_first_positive_rank": 1.028,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 244.6973513062373,
|
|
"median_score_margin": 263.6290616989136,
|
|
"negative_margin_rate": 0.0275,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2000
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": null,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": null,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": null,
|
|
"counts": {
|
|
"tp": 2000,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.6521499448732084,
|
|
"recall1": 0.6521499448732084,
|
|
"recall3": 0.7761852260198456,
|
|
"recall5": 0.8456449834619625,
|
|
"recall8": 0.9162072767364939,
|
|
"evidence_count_in_top3": 0.7761852260198456,
|
|
"hit3": 0.7761852260198456,
|
|
"mrr": 0.7410856875575729,
|
|
"ndcg3": 0.7240558854681862,
|
|
"mean_first_positive_rank": 2.54244762954796,
|
|
"all_evidence_in_top3": 0.7761852260198456,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 38.1325886073587,
|
|
"median_score_margin": 54.42159175872803,
|
|
"negative_margin_rate": 0.3478500551267916,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9999999804822718,
|
|
"unknown_mean_need_prob": 0.0001597598329493619,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2000,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 2000,
|
|
"top1": null,
|
|
"recall1": null,
|
|
"recall3": null,
|
|
"recall5": null,
|
|
"recall8": null,
|
|
"evidence_count_in_top3": null,
|
|
"hit3": null,
|
|
"mrr": null,
|
|
"ndcg3": null,
|
|
"mean_first_positive_rank": null,
|
|
"all_evidence_in_top3": null,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": null,
|
|
"hop_under_prediction": null,
|
|
"hop_over_prediction": null,
|
|
"mean_score_margin": null,
|
|
"median_score_margin": null,
|
|
"negative_margin_rate": null,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 2000
|
|
},
|
|
"known_mean_need_prob": null,
|
|
"unknown_mean_need_prob": 2.5479757829358922e-15,
|
|
"thr0.50": {
|
|
"need_precision": null,
|
|
"need_recall": null,
|
|
"need_f1": null,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": null,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 2000,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} |