{ "V2-128 deployed(v3)": { "checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_qwen_router_entities\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 8157561 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 8157561 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.155849999122438, "single_query_latency_ms_p50": 1.155849999122438, "single_query_latency_ms_p95": 1.1732999992091209, "queries_per_second": 865.1641655571528, "batched_qps_64": 54038.53621622001, "batched_qps_256": 199554.12120495233, "batched_latency_ms_64": 0.01850531250511267, "samples": 5, "source": "bench_router_latency.py (interleaved round-robin medians)" }, "cpu": { "single_query_latency_ms_mean": 0.38441666644454625, "single_query_latency_ms_p50": 0.36130000080447644, "single_query_latency_ms_p95": 0.5763999979535583, "queries_per_second": 2601.3440292507566, "samples": 30, "batched_qps_64": 12592.97154659022, "batched_latency_ms_64": 0.07940937500734435, "batched_qps_256": 13508.49346529746, "batched_latency_ms_256": 0.07402749999982916 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.4112495789828225, "recall1": 0.37027057370607386, "recall3": 0.4673009992141013, "recall5": 0.4891377568204783, "recall8": 0.5160267205568654, "evidence_count_in_top3": 0.5420455821264174, "hit3": 0.5003368137420007, "mrr": 0.476908110965675, "ndcg3": 0.44220238482445295, "mean_first_positive_rank": 14.38671831144044, "all_evidence_in_top3": 0.43426518468620184, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.3715, "hop_accuracy": 0.7322992700729927, "hop_accuracy_answerable": 0.9010890310991355, "hop_under_prediction": 0.046817110138093636, "hop_over_prediction": 0.05209385876277085, "mean_score_margin": -9.29892828391031, "median_score_margin": -9.269989609718323, "negative_margin_rate": 0.5887504210171774, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 4042, "true_0_pred_2": 64, "true_1_pred_0": 834, "true_1_pred_1": 14052, "true_1_pred_2": 928, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9762653562039442, "unknown_mean_need_prob": 0.998855464207621, "thr0.30": { "need_precision": 0.8126824817518248, "need_recall": 1.0, "need_f1": 0.8966628076710123, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8126824817518248, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 17814, "tn": 0, "fp": 4106, "fn": 0 } }, "thr0.40": { "need_precision": 0.8126824817518248, "need_recall": 1.0, "need_f1": 0.8966628076710123, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8126824817518248, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 17814, "tn": 0, "fp": 4106, "fn": 0 } }, "thr0.50": { "need_precision": 0.8124086257309941, "need_recall": 0.998203660042663, "need_f1": 0.895773512669387, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8112226277372263, "known_question_refusal_rate": 0.001796339957336926, "unknown_question_read_rate": 1.0, "counts": { "tp": 17782, "tn": 0, "fp": 4106, "fn": 32 } }, "thr0.60": { "need_precision": 0.8115476409032495, "need_recall": 0.9925900976759852, "need_f1": 0.892985202767537, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8066605839416059, "known_question_refusal_rate": 0.00740990232401482, "unknown_question_read_rate": 1.0, "counts": { "tp": 17682, "tn": 0, "fp": 4106, "fn": 132 } }, "thr0.70": { "need_precision": 0.8112270700197692, "need_recall": 0.9905130796003143, "need_f1": 0.8919499557689878, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8049726277372263, "known_question_refusal_rate": 0.00948692039968564, "unknown_question_read_rate": 1.0, "counts": { "tp": 17645, "tn": 0, "fp": 4106, "fn": 169 } }, "thr0.80": { "need_precision": 0.8106525247867189, "need_recall": 0.9868081284383069, "need_f1": 0.8900984835059115, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8019616788321168, "known_question_refusal_rate": 0.01319187156169305, "unknown_question_read_rate": 1.0, "counts": { "tp": 17579, "tn": 0, "fp": 4106, "fn": 235 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_2": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.9999341862276196, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 1.0, "counts": { "tp": 0, "tn": 0, "fp": 64, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.442375, "recall1": 0.39675, "recall3": 0.49153125, "recall5": 0.50303125, "recall8": 0.51234375, "evidence_count_in_top3": 0.57475, "hit3": 0.5283125, "mrr": 0.500402578720693, "ndcg3": 0.4692792057097287, "mean_first_positive_rank": 14.9525, "all_evidence_in_top3": 0.45475, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.3715, "hop_accuracy": 0.74955, "hop_accuracy_answerable": 0.9369375, "hop_under_prediction": 0.0160625, "hop_over_prediction": 0.047, "mean_score_margin": -8.091850390457548, "median_score_margin": -6.662378132343292, "negative_margin_rate": 0.557625, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 4000, "true_0_pred_2": 0, "true_1_pred_0": 257, "true_1_pred_1": 12991, "true_1_pred_2": 752, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9821553228273988, "unknown_mean_need_prob": 0.9988265322595835, "thr0.50": { "need_precision": 0.8, "need_recall": 1.0, "need_f1": 0.888888888888889, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 16000, "tn": 0, "fp": 4000, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.14528412419449327, "recall1": 0.14528412419449327, "recall3": 0.2677211482132396, "recall5": 0.38488576449912126, "recall8": 0.5653192735793791, "evidence_count_in_top3": 0.2677211482132396, "hit3": 0.2677211482132396, "mrr": 0.2801716446629072, "ndcg3": 0.21516992510461316, "mean_first_positive_rank": 9.195664909197422, "all_evidence_in_top3": 0.2677211482132396, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.5740740740740741, "hop_accuracy_answerable": 0.5811364967779731, "hop_under_prediction": 0.3380199179847686, "hop_over_prediction": 0.08084358523725835, "mean_score_margin": -19.359866383266713, "median_score_margin": -18.159542322158813, "negative_margin_rate": 0.8547158758055068, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 21, "true_0_pred_2": 0, "true_1_pred_0": 577, "true_1_pred_1": 992, "true_1_pred_2": 138 }, "known_mean_need_prob": 0.9195722371566764, "unknown_mean_need_prob": 0.9999671209426153, "thr0.50": { "need_precision": 0.9876179245283019, "need_recall": 0.981253661394259, "need_f1": 0.9844255069056715, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.9693287037037037, "known_question_refusal_rate": 0.018746338605741066, "unknown_question_read_rate": 1.0, "counts": { "tp": 1675, "tn": 0, "fp": 21, "fn": 32 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.0, "recall1": 0.0, "recall3": 0.028037383177570093, "recall5": 0.07476635514018691, "recall8": 0.2803738317757009, "evidence_count_in_top3": 0.028037383177570093, "hit3": 0.028037383177570093, "mrr": 0.1023068389893556, "ndcg3": 0.01524233414552764, "mean_first_positive_rank": 12.598130841121495, "all_evidence_in_top3": 0.028037383177570093, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.5390625, "hop_accuracy_answerable": 0.6448598130841121, "hop_under_prediction": 0.0, "hop_over_prediction": 0.35514018691588783, "mean_score_margin": -29.291684916086286, "median_score_margin": -29.07153034210205, "negative_margin_rate": 1.0, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 21, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 69, "true_1_pred_2": 38 }, "known_mean_need_prob": 0.9999633771236812, "unknown_mean_need_prob": 0.9999671209426153, "thr0.50": { "need_precision": 0.8359375, "need_recall": 1.0, "need_f1": 0.9106382978723404, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8359375, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 107, "tn": 0, "fp": 21, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.796, "recall1": 0.796, "recall3": 0.9125, "recall5": 0.919, "recall8": 0.927, "evidence_count_in_top3": 0.9125, "hit3": 0.9125, "mrr": 0.8582544951434297, "ndcg3": 0.8678012294946459, "mean_first_positive_rank": 2.8275, "all_evidence_in_top3": 0.9125, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.4955, "hop_accuracy_answerable": 0.4955, "hop_under_prediction": 0.1285, "hop_over_prediction": 0.376, "mean_score_margin": 21.036133953779935, "median_score_margin": 21.400100469589233, "negative_margin_rate": 0.204, "hop_confusion": { "true_1_pred_0": 257, "true_1_pred_1": 991, "true_1_pred_2": 752 }, "known_mean_need_prob": 0.9764632563889026, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.231, "recall1": 0.231, "recall3": 0.2535, "recall5": 0.255, "recall8": 0.257, "evidence_count_in_top3": 0.2535, "hit3": 0.2535, "mrr": 0.26603145346142226, "ndcg3": 0.24486859507142916, "mean_first_positive_rank": 23.789, "all_evidence_in_top3": 0.2535, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -27.718347060799598, "median_score_margin": -38.59446668624878, "negative_margin_rate": 0.769, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9723067943453789, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.2255, "recall1": 0.2255, "recall3": 0.2455, "recall5": 0.2475, "recall8": 0.2485, "evidence_count_in_top3": 0.2455, "hit3": 0.2455, "mrr": 0.2596338049221357, "ndcg3": 0.23792220044107196, "mean_first_positive_rank": 24.0695, "all_evidence_in_top3": 0.2455, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -28.903205164782705, "median_score_margin": -39.01692509651184, "negative_margin_rate": 0.7745, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9724746745526791, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.99774328109622, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 1.0, "counts": { "tp": 0, "tn": 0, "fp": 2000, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.5255, "recall1": 0.5255, "recall3": 0.6245, "recall5": 0.6265, "recall8": 0.6295, "evidence_count_in_top3": 0.6245, "hit3": 0.6245, "mrr": 0.5855216777342396, "ndcg3": 0.585867169546431, "mean_first_positive_rank": 12.2085, "all_evidence_in_top3": 0.6245, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -1.0039909677058458, "median_score_margin": 2.929476022720337, "negative_margin_rate": 0.4745, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9875293919146061, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.73, "recall1": 0.365, "recall3": 0.66575, "recall5": 0.72725, "recall8": 0.77375, "evidence_count_in_top3": 1.3315, "hit3": 0.96, "mrr": 0.84532448592014, "ndcg3": 0.6658792959224611, "mean_first_positive_rank": 1.59, "all_evidence_in_top3": 0.3715, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.3715, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.786984693795443, "median_score_margin": 5.27826452255249, "negative_margin_rate": 0.27, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9993210740983486, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.443, "recall1": 0.443, "recall3": 0.5525, "recall5": 0.5615, "recall8": 0.5665, "evidence_count_in_top3": 0.5525, "hit3": 0.5525, "mrr": 0.5118924660255056, "ndcg3": 0.5092063534375025, "mean_first_positive_rank": 13.8255, "all_evidence_in_top3": 0.5525, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -2.995655853047967, "median_score_margin": -5.5251545906066895, "negative_margin_rate": 0.557, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9880377069413662, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.249, "recall1": 0.249, "recall3": 0.287, "recall5": 0.293, "recall8": 0.3005, "evidence_count_in_top3": 0.287, "hit3": 0.287, "mrr": 0.2919631936667751, "ndcg3": 0.27173149797678653, "mean_first_positive_rank": 22.067, "all_evidence_in_top3": 0.287, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -19.72415616109967, "median_score_margin": -21.928287506103516, "negative_margin_rate": 0.751, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9656237106621265, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.339, "recall1": 0.339, "recall3": 0.391, "recall5": 0.3945, "recall8": 0.396, "evidence_count_in_top3": 0.391, "hit3": 0.391, "mrr": 0.3845990528918967, "ndcg3": 0.3709573037875013, "mean_first_positive_rank": 19.243, "all_evidence_in_top3": 0.391, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -8.212566563799978, "median_score_margin": -16.937832549214363, "negative_margin_rate": 0.661, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9954859737157822, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.13671444321940462, "recall1": 0.13671444321940462, "recall3": 0.2535832414553473, "recall5": 0.36659316427783906, "recall8": 0.5485115766262404, "evidence_count_in_top3": 0.2535832414553473, "hit3": 0.2535832414553473, "mrr": 0.26968017045834825, "ndcg3": 0.2033770627933551, "mean_first_positive_rank": 9.396361631753033, "all_evidence_in_top3": 0.2535832414553473, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.5526041666666667, "hop_accuracy_answerable": 0.5848952590959207, "hop_under_prediction": 0.31808158765159866, "hop_over_prediction": 0.0970231532524807, "mean_score_margin": -19.945701324287494, "median_score_margin": -18.862584590911865, "negative_margin_rate": 0.8632855567805954, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 42, "true_0_pred_2": 64, "true_1_pred_0": 577, "true_1_pred_1": 1061, "true_1_pred_2": 176 }, "known_mean_need_prob": 0.9243141621712682, "unknown_mean_need_prob": 0.9999472358316746, "thr0.50": { "need_precision": 0.9438559322033898, "need_recall": 0.9823594266813671, "need_f1": 0.9627228525121555, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.928125, "known_question_refusal_rate": 0.017640573318632856, "unknown_question_read_rate": 1.0, "counts": { "tp": 1782, "tn": 0, "fp": 106, "fn": 32 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.9999097834229469, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 1.0, "counts": { "tp": 0, "tn": 0, "fp": 2000, "fn": 0 } } } } }, "V2-128 v6 best": { "checkpoint": "checkpoints\\router_v6_v2_128\\router_best.pt", "router": { "kind": "router_v2", "step": 6000, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 24472973 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 24472973 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.167899999927613, "single_query_latency_ms_p50": 1.167899999927613, "single_query_latency_ms_p95": 1.196150000396301, "queries_per_second": 856.2376916362535, "batched_qps_64": 54149.18098951274, "batched_qps_256": 196829.20449441153, "batched_latency_ms_64": 0.018467500001406734, "samples": 5, "source": "bench_router_latency.py (interleaved round-robin medians)" }, "cpu": { "single_query_latency_ms_mean": 0.33882666709056747, "single_query_latency_ms_p50": 0.338400001055561, "single_query_latency_ms_p95": 0.4341000021668151, "queries_per_second": 2951.3615577746205, "samples": 30, "batched_qps_64": 12676.831899779869, "batched_latency_ms_64": 0.07888406250913249, "batched_qps_256": 13789.017263119036, "batched_latency_ms_256": 0.07252148437544292 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.946165936903559, "recall1": 0.8919108566296171, "recall3": 0.9780509711462895, "recall5": 0.984001347254968, "recall8": 0.9933198607836533, "evidence_count_in_top3": 1.0897608622431796, "hit3": 0.9786123273829572, "mrr": 0.9648813461092199, "ndcg3": 0.9649494672503449, "mean_first_positive_rank": 1.1781744695183562, "all_evidence_in_top3": 0.9774896149096216, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.99, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 14.959203617273067, "median_score_margin": 14.255205273628235, "negative_margin_rate": 0.053834063096441, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999601252354449, "unknown_mean_need_prob": 0.0012111918483046415, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.010367806648901023, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.976875, "recall1": 0.91646875, "recall3": 0.999375, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.12375, "hit3": 1.0, "mrr": 0.9883645833333333, "ndcg3": 0.9901670270848972, "mean_first_positive_rank": 1.0235625, "all_evidence_in_top3": 0.99875, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.99, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 16.402275828796906, "median_score_margin": 15.20901894569397, "negative_margin_rate": 0.023125, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999701014049351, "unknown_mean_need_prob": 6.905754663642937e-05, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.6854130052724078, "recall1": 0.6854130052724078, "recall3": 0.786174575278266, "recall5": 0.836555360281195, "recall8": 0.9302870533099004, "evidence_count_in_top3": 0.786174575278266, "hit3": 0.786174575278266, "mrr": 0.7618258791916341, "ndcg3": 0.7445377808477717, "mean_first_positive_rank": 2.570591681312244, "all_evidence_in_top3": 0.786174575278266, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.378015720335124, "median_score_margin": 3.452073097229004, "negative_margin_rate": 0.3145869947275923, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9998671145696143, "unknown_mean_need_prob": 0.0960329504062732, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.514018691588785, "recall1": 0.514018691588785, "recall3": 0.8504672897196262, "recall5": 0.9439252336448598, "recall8": 1.0, "evidence_count_in_top3": 0.8504672897196262, "hit3": 0.8504672897196262, "mrr": 0.6927681352914997, "ndcg3": 0.7103867694592852, "mean_first_positive_rank": 2.0841121495327104, "all_evidence_in_top3": 0.8504672897196262, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.11619492766456069, "median_score_margin": 0.07718276977539062, "negative_margin_rate": 0.48598130841121495, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 0.9999521859338351, "unknown_mean_need_prob": 0.0960329504062732, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9885, "recall1": 0.9885, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99425, "ndcg3": 0.9957556921660717, "mean_first_positive_rank": 1.0115, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 18.511014288797973, "median_score_margin": 17.0071280002594, "negative_margin_rate": 0.0115, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999475074112415, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9755, "recall1": 0.9755, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98775, "ndcg3": 0.9909577789625007, "mean_first_positive_rank": 1.0245, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 17.432058019019664, "median_score_margin": 17.119813442230225, "negative_margin_rate": 0.0245, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.999997846186161, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9805, "recall1": 0.9805, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99025, "ndcg3": 0.9928031301946435, "mean_first_positive_rank": 1.0195, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 18.200293550987727, "median_score_margin": 18.367231130599976, "negative_margin_rate": 0.0195, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999978821873665, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.00013719209501925888, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.98, "recall1": 0.98, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9899166666666667, "ndcg3": 0.9925531301946434, "mean_first_positive_rank": 1.0205, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 17.648428340390325, "median_score_margin": 17.865666389465332, "negative_margin_rate": 0.02, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999990122318267, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9665, "recall1": 0.48325, "recall3": 0.995, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.99, "hit3": 1.0, "mrr": 0.983, "ndcg3": 0.9775122185202447, "mean_first_positive_rank": 1.035, "all_evidence_in_top3": 0.99, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.99, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 10.340590946737677, "median_score_margin": 12.604959726333618, "negative_margin_rate": 0.0335, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999316669106484, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9855, "recall1": 0.9855, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99275, "ndcg3": 0.9946484814267861, "mean_first_positive_rank": 1.0145, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 16.270763264942914, "median_score_margin": 14.272640228271484, "negative_margin_rate": 0.0145, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.999909707903862, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.978, "recall1": 0.978, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.989, "ndcg3": 0.9918804545785721, "mean_first_positive_rank": 1.022, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 17.229778190023264, "median_score_margin": 17.07593348622322, "negative_margin_rate": 0.022, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999774325489997, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9605, "recall1": 0.9605, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98, "ndcg3": 0.9852253306357154, "mean_first_positive_rank": 1.041, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 15.585280029475689, "median_score_margin": 15.148456335067749, "negative_margin_rate": 0.0395, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.999999755859375, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.675303197353914, "recall1": 0.675303197353914, "recall3": 0.7899669239250275, "recall5": 0.8428886438809261, "recall8": 0.934399117971334, "evidence_count_in_top3": 0.7899669239250275, "hit3": 0.7899669239250275, "mrr": 0.7577524621038092, "ndcg3": 0.74252336066113, "mean_first_positive_rank": 2.541896361631753, "all_evidence_in_top3": 0.7899669239250275, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.2308930415391117, "median_score_margin": 3.026643395423889, "negative_margin_rate": 0.324696802646086, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9998721325607783, "unknown_mean_need_prob": 0.04431059945842585, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 9.229982535998715e-07, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "V2-128 v6 final": { "checkpoint": "checkpoints\\router_v6_v2_128\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 8156921 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 8156921 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.165349998700549, "single_query_latency_ms_p50": 1.165349998700549, "single_query_latency_ms_p95": 1.1786999984906288, "queries_per_second": 858.1112979920828, "batched_qps_64": 53471.46796235134, "batched_qps_256": 198025.9290094514, "batched_latency_ms_64": 0.018701562498790736, "samples": 5, "source": "bench_router_latency.py (interleaved round-robin medians)" }, "cpu": { "single_query_latency_ms_mean": 0.3544133333586312, "single_query_latency_ms_p50": 0.3417000007175375, "single_query_latency_ms_p95": 0.4805999997188337, "queries_per_second": 2821.564275032788, "samples": 30, "batched_qps_64": 13137.043999093688, "batched_latency_ms_64": 0.07612062501038963, "batched_qps_256": 13174.159298667208, "batched_latency_ms_256": 0.07590617187247517 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9436959694622207, "recall1": 0.8882059054676097, "recall3": 0.9639609296059279, "recall5": 0.973981138430448, "recall8": 0.9864151790726395, "evidence_count_in_top3": 1.0717974626698104, "hit3": 0.9683956438756035, "mrr": 0.9597054333626427, "ndcg3": 0.9530820526370456, "mean_first_positive_rank": 1.2573818345121814, "all_evidence_in_top3": 0.9595262153362524, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.921, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 51.315328869764656, "median_score_margin": 56.234654903411865, "negative_margin_rate": 0.05630403053777928, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999984274064, "unknown_mean_need_prob": 4.836221294362452e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 4.8406160974828055e-06, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.9838125, "recall1": 0.92203125, "recall3": 0.9950625, "recall5": 0.99978125, "recall8": 0.999875, "evidence_count_in_top3": 1.115125, "hit3": 1.0, "mrr": 0.991875, "ndcg3": 0.9876817889701721, "mean_first_positive_rank": 1.016375, "all_evidence_in_top3": 0.990125, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.921, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 57.110110210259094, "median_score_margin": 59.90444374084473, "negative_margin_rate": 0.0161875, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999701977, "unknown_mean_need_prob": 6.379481702447893e-11, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.5940246045694201, "recall1": 0.5940246045694201, "recall3": 0.6842413591095489, "recall5": 0.7352079671939075, "recall8": 0.8611599297012302, "evidence_count_in_top3": 0.6842413591095489, "hit3": 0.6842413591095489, "mrr": 0.6760273585431921, "ndcg3": 0.6466497456649108, "mean_first_positive_rank": 3.4463971880492092, "all_evidence_in_top3": 0.6842413591095489, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.27154229221702947, "median_score_margin": 7.975786209106445, "negative_margin_rate": 0.40597539543058, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999838679872, "unknown_mean_need_prob": 0.0004654159529796483, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.5233644859813084, "recall1": 0.5233644859813084, "recall3": 0.7757009345794392, "recall5": 0.9252336448598131, "recall8": 0.9719626168224299, "evidence_count_in_top3": 0.7757009345794392, "hit3": 0.7757009345794392, "mrr": 0.674896157840083, "ndcg3": 0.6678873486315127, "mean_first_positive_rank": 2.3738317757009346, "all_evidence_in_top3": 0.7757009345794392, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.8786688875929217, "median_score_margin": 0.4756507873535156, "negative_margin_rate": 0.4766355140186916, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0004654159529796483, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.993, "recall1": 0.993, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9965, "ndcg3": 0.9974165082750002, "mean_first_positive_rank": 1.007, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 54.4210530708991, "median_score_margin": 56.92102670669556, "negative_margin_rate": 0.007, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.975, "recall1": 0.975, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9875, "ndcg3": 0.9907732438392864, "mean_first_positive_rank": 1.025, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 64.29146421498805, "median_score_margin": 73.40720510482788, "negative_margin_rate": 0.025, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9845, "recall1": 0.9845, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99225, "ndcg3": 0.9942794111803576, "mean_first_positive_rank": 1.0155, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 65.52974343589041, "median_score_margin": 73.97303533554077, "negative_margin_rate": 0.0155, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.275862421441728e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9795, "recall1": 0.9795, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9896666666666666, "ndcg3": 0.9923685950714292, "mean_first_positive_rank": 1.021, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 60.150181606799364, "median_score_margin": 68.01980018615723, "negative_margin_rate": 0.0205, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9885, "recall1": 0.49425, "recall3": 0.9605, "recall5": 0.99825, "recall8": 0.999, "evidence_count_in_top3": 1.921, "hit3": 1.0, "mrr": 0.99425, "ndcg3": 0.9452009954703022, "mean_first_positive_rank": 1.0115, "all_evidence_in_top3": 0.921, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.921, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 55.6439196386747, "median_score_margin": 62.50904202461243, "negative_margin_rate": 0.0115, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999997615814, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.995, "recall1": 0.995, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9975, "ndcg3": 0.9981546487678573, "mean_first_positive_rank": 1.005, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 48.13904330630228, "median_score_margin": 49.25835180282593, "negative_margin_rate": 0.005, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9805, "recall1": 0.9805, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9901666666666666, "ndcg3": 0.9927376653178577, "mean_first_positive_rank": 1.02, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 50.85701001610607, "median_score_margin": 54.9089093208313, "negative_margin_rate": 0.0195, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9745, "recall1": 0.9745, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9871666666666666, "ndcg3": 0.9905232438392865, "mean_first_positive_rank": 1.026, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 57.84846639241278, "median_score_margin": 63.093711376190186, "negative_margin_rate": 0.0255, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.5898566703417861, "recall1": 0.5898566703417861, "recall3": 0.6896361631753032, "recall5": 0.7464167585446527, "recall8": 0.8676957001102535, "evidence_count_in_top3": 0.6896361631753032, "hit3": 0.6896361631753032, "mrr": 0.6759606339151697, "ndcg3": 0.6479024598421029, "mean_first_positive_rank": 3.383131201764057, "all_evidence_in_top3": 0.6896361631753032, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.20369631854576997, "median_score_margin": 6.827788352966309, "negative_margin_rate": 0.4101433296582139, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999848195447, "unknown_mean_need_prob": 0.00018733273071117103, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 3.3919047850488803e-15, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "XL-128 v6 best": { "checkpoint": "checkpoints\\router_v6_xl128\\router_best.pt", "router": { "kind": "router_xl", "step": 1000, "arch_config": { "arch": "router_xl", "arch_version": 1, "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3, "encoder_layers": 2, "encoder_hidden": 512, "pair_blocks": 1, "pair_hidden": 512, "pair_expansion": 2, "pair_dropout": 0.05, "use_interaction": true, "policy_layers": 2, "policy_hidden": 512, "policy_dropout": 0.0, "learnable_cosine_scale": true }, "parameters": 6714639, "checkpoint_bytes": 81666921 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 81666921 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.578349998453632, "single_query_latency_ms_p50": 1.578349998453632, "single_query_latency_ms_p95": 1.6234499998972751, "queries_per_second": 633.5730357523598, "batched_qps_64": 38630.547104467114, "batched_qps_256": 141500.569342867, "batched_latency_ms_64": 0.02588625000043976, "samples": 5, "source": "bench_router_latency.py (interleaved round-robin medians)" }, "cpu": { "single_query_latency_ms_mean": 1.4055266667128308, "single_query_latency_ms_p50": 1.3804000009258743, "single_query_latency_ms_p95": 1.7585999994480517, "queries_per_second": 711.4770738137224, "samples": 30, "batched_qps_64": 2431.7772491816463, "batched_latency_ms_64": 0.4112218750037755, "batched_qps_256": 2460.1366567422874, "batched_latency_ms_256": 0.4064814843758313 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9566071629055799, "recall1": 0.9027730998091389, "recall3": 0.9874817559223082, "recall5": 0.9922532839339845, "recall8": 0.9991018300213316, "evidence_count_in_top3": 1.0992477826428653, "hit3": 0.9879869765353093, "mrr": 0.9737093504776227, "ndcg3": 0.975110976145454, "mean_first_positive_rank": 1.098574155158864, "all_evidence_in_top3": 0.9869765353093073, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.991, "hop_accuracy": 0.9995437956204379, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 16.798009918724365, "median_score_margin": 17.34114497900009, "negative_margin_rate": 0.04339283709442012, "hop_confusion": { "true_0_pred_0": 4096, "true_0_pred_1": 10, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9900794763436311, "unknown_mean_need_prob": 0.036242366845311694, "thr0.30": { "need_precision": 0.9964201812283253, "need_recall": 1.0, "need_f1": 0.9982068810938025, "specificity_unknown_refusal": 0.9844130540672187, "abstention_accuracy": 0.997080291970803, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.015586945932781296, "counts": { "tp": 17814, "tn": 4042, "fp": 64, "fn": 0 } }, "thr0.40": { "need_precision": 0.998150949739452, "need_recall": 1.0, "need_f1": 0.9990746193320434, "specificity_unknown_refusal": 0.9919629810034096, "abstention_accuracy": 0.9984945255474452, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.008037018996590355, "counts": { "tp": 17814, "tn": 4073, "fp": 33, "fn": 0 } }, "thr0.50": { "need_precision": 0.9990466042285907, "need_recall": 1.0, "need_f1": 0.9995230747650443, "specificity_unknown_refusal": 0.995859717486605, "abstention_accuracy": 0.9992244525547446, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.004140282513395032, "counts": { "tp": 17814, "tn": 4089, "fp": 17, "fn": 0 } }, "thr0.60": { "need_precision": 0.999326825984517, "need_recall": 1.0, "need_f1": 0.9996632996632996, "specificity_unknown_refusal": 0.9970774476376035, "abstention_accuracy": 0.9994525547445255, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0029225523623964927, "counts": { "tp": 17814, "tn": 4094, "fp": 12, "fn": 0 } }, "thr0.70": { "need_precision": 0.9996632996632997, "need_recall": 1.0, "need_f1": 0.9998316214850985, "specificity_unknown_refusal": 0.9985387238188017, "abstention_accuracy": 0.9997262773722628, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0014612761811982464, "counts": { "tp": 17814, "tn": 4100, "fp": 6, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.3642658165190369, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.953125, "abstention_accuracy": 0.953125, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.046875, "counts": { "tp": 0, "tn": 61, "fp": 3, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.9765625, "recall1": 0.916625, "recall3": 0.9994375, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.123875, "hit3": 1.0, "mrr": 0.9882291666666666, "ndcg3": 0.9904076605853642, "mean_first_positive_rank": 1.02375, "all_evidence_in_top3": 0.998875, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.991, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 17.91327105785167, "median_score_margin": 18.24349021911621, "negative_margin_rate": 0.0234375, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9914327386058867, "unknown_mean_need_prob": 0.027471872764872386, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.8060925600468658, "recall1": 0.8060925600468658, "recall3": 0.8840070298769771, "recall5": 0.9197422378441711, "recall8": 0.9906268306971294, "evidence_count_in_top3": 0.8840070298769771, "hit3": 0.8840070298769771, "mrr": 0.8601396423013294, "ndcg3": 0.8521063487572197, "mean_first_positive_rank": 1.7311072056239016, "all_evidence_in_top3": 0.8840070298769771, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.9971064814814815, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 7.396975287374308, "median_score_margin": 7.794462203979492, "negative_margin_rate": 0.19390743995313414, "hop_confusion": { "true_0_pred_0": 16, "true_0_pred_1": 5, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.98037854359034, "unknown_mean_need_prob": 0.37168226071766447, "thr0.50": { "need_precision": 0.9959159859976663, "need_recall": 1.0, "need_f1": 0.997953814674072, "specificity_unknown_refusal": 0.6666666666666666, "abstention_accuracy": 0.9959490740740741, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.3333333333333333, "counts": { "tp": 1707, "tn": 14, "fp": 7, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.37383177570093457, "recall1": 0.37383177570093457, "recall3": 0.8504672897196262, "recall5": 0.9906542056074766, "recall8": 1.0, "evidence_count_in_top3": 0.8504672897196262, "hit3": 0.8504672897196262, "mrr": 0.6143302180685358, "ndcg3": 0.6500824519693008, "mean_first_positive_rank": 2.196261682242991, "all_evidence_in_top3": 0.8504672897196262, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.9609375, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.007242532534019969, "median_score_margin": -0.2684917449951172, "negative_margin_rate": 0.6261682242990654, "hop_confusion": { "true_0_pred_0": 16, "true_0_pred_1": 5, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 0.9424841119864277, "unknown_mean_need_prob": 0.37168226071766447, "thr0.50": { "need_precision": 0.9385964912280702, "need_recall": 1.0, "need_f1": 0.9683257918552036, "specificity_unknown_refusal": 0.6666666666666666, "abstention_accuracy": 0.9453125, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.3333333333333333, "counts": { "tp": 107, "tn": 14, "fp": 7, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9785, "recall1": 0.9785, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98925, "ndcg3": 0.9920649897017864, "mean_first_positive_rank": 1.0215, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 19.512430111046882, "median_score_margin": 19.13399839401245, "negative_margin_rate": 0.0215, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9832325124442577, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.978, "recall1": 0.978, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.989, "ndcg3": 0.9918804545785721, "mean_first_positive_rank": 1.022, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 18.416588767559734, "median_score_margin": 18.676148295402527, "negative_margin_rate": 0.022, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9980810531675816, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.985, "recall1": 0.985, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9924166666666666, "ndcg3": 0.9943984814267861, "mean_first_positive_rank": 1.0155, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 18.82138747824775, "median_score_margin": 18.756187587976456, "negative_margin_rate": 0.015, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9980952816009522, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.05116949350852519, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9765, "recall1": 0.9765, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98825, "ndcg3": 0.9913268492089293, "mean_first_positive_rank": 1.0235, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 15.71517678581737, "median_score_margin": 16.085471630096436, "negative_margin_rate": 0.0235, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9980619028508663, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.959, "recall1": 0.4795, "recall3": 0.9955, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.991, "hit3": 1.0, "mrr": 0.9793333333333334, "ndcg3": 0.9775264704150523, "mean_first_positive_rank": 1.042, "all_evidence_in_top3": 0.991, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.991, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 17.043865082176403, "median_score_margin": 20.163950562477112, "negative_margin_rate": 0.041, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9835184065401554, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.987, "recall1": 0.987, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9935, "ndcg3": 0.995202086796429, "mean_first_positive_rank": 1.013, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 18.87393392176647, "median_score_margin": 18.269309908151627, "negative_margin_rate": 0.013, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9775506886839866, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.977, "recall1": 0.977, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9885, "ndcg3": 0.9915113843321435, "mean_first_positive_rank": 1.023, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 17.55705898122629, "median_score_margin": 17.530375365167856, "negative_margin_rate": 0.023, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9958070812821388, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9715, "recall1": 0.9715, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9855833333333334, "ndcg3": 0.9893505682232151, "mean_first_positive_rank": 1.0295, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 17.365727334972473, "median_score_margin": 16.746063470840454, "negative_margin_rate": 0.0285, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9971149822771549, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.7805953693495039, "recall1": 0.7805953693495039, "recall3": 0.8820286659316428, "recall5": 0.9239250275633958, "recall8": 0.9911797133406836, "evidence_count_in_top3": 0.8820286659316428, "hit3": 0.8820286659316428, "mrr": 0.8456404094496708, "ndcg3": 0.8401898344483404, "mean_first_positive_rank": 1.7585446527012127, "all_evidence_in_top3": 0.8820286659316428, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.9947916666666666, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 6.961086971625735, "median_score_margin": 7.221376895904541, "negative_margin_rate": 0.21940463065049615, "hop_confusion": { "true_0_pred_0": 96, "true_0_pred_1": 10, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9781433152653022, "unknown_mean_need_prob": 0.3672044076166063, "thr0.50": { "need_precision": 0.9907154560349536, "need_recall": 1.0, "need_f1": 0.9953360768175583, "specificity_unknown_refusal": 0.839622641509434, "abstention_accuracy": 0.9911458333333333, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.16037735849056603, "counts": { "tp": 1814, "tn": 89, "fp": 17, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.0037742520212195813, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "XL-128 v6 final": { "checkpoint": "checkpoints\\router_v6_xl128\\memory_router_xl.pt", "router": { "kind": "router_xl", "step": null, "arch_config": { "arch": "router_xl", "arch_version": 1, "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3, "encoder_layers": 2, "encoder_hidden": 512, "pair_blocks": 1, "pair_hidden": 512, "pair_expansion": 2, "pair_dropout": 0.05, "use_interaction": true, "policy_layers": 2, "policy_hidden": 512, "policy_dropout": 0.0, "learnable_cosine_scale": true }, "parameters": 6714639, "checkpoint_bytes": 27922475 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 27922475 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.5958999993017642, "single_query_latency_ms_p50": 1.5958999993017642, "single_query_latency_ms_p95": 1.6187000001082197, "queries_per_second": 626.6056773215856, "batched_qps_64": 36843.43841344889, "batched_qps_256": 134976.95897715187, "batched_latency_ms_64": 0.027141875000324944, "samples": 5, "source": "bench_router_latency.py (interleaved round-robin medians)" }, "cpu": { "single_query_latency_ms_mean": 1.4327199997448286, "single_query_latency_ms_p50": 1.4391999975487124, "single_query_latency_ms_p95": 1.6943999980867375, "queries_per_second": 697.9730862821087, "samples": 30, "batched_qps_64": 2507.9254362757065, "batched_latency_ms_64": 0.3987359374946209, "batched_qps_256": 2349.3282205997198, "batched_latency_ms_256": 0.4256535937514627 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9518917705175705, "recall1": 0.8963736387111261, "recall3": 0.9745424946671157, "recall5": 0.9820366004266308, "recall8": 0.9912709105198159, "evidence_count_in_top3": 1.0851577411025037, "hit3": 0.9761984955652857, "mrr": 0.9670023550641874, "ndcg3": 0.9642642772784563, "mean_first_positive_rank": 1.1905243067250477, "all_evidence_in_top3": 0.9728864937689458, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9705, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 200.33457717921095, "median_score_margin": 219.4529800415039, "negative_margin_rate": 0.04810822948242955, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999979790499, "unknown_mean_need_prob": 3.8975788264237145e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 3.895038815020929e-06, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.984375, "recall1": 0.9225625, "recall3": 0.99815625, "recall5": 0.9995, "recall8": 0.99959375, "evidence_count_in_top3": 1.1213125, "hit3": 1.0, "mrr": 0.9921354166666666, "ndcg3": 0.9914167841247224, "mean_first_positive_rank": 1.0159375, "all_evidence_in_top3": 0.9963125, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9705, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 217.4091286659348, "median_score_margin": 228.10742282867432, "negative_margin_rate": 0.015625, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999627471, "unknown_mean_need_prob": 6.477803048274694e-11, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.6754540128881078, "recall1": 0.6754540128881078, "recall3": 0.7592267135325131, "recall5": 0.8189806678383128, "recall8": 0.9127123608670181, "evidence_count_in_top3": 0.7592267135325131, "hit3": 0.7592267135325131, "mrr": 0.7481470978152184, "ndcg3": 0.7247037236923608, "mean_first_positive_rank": 2.7809021675454013, "all_evidence_in_top3": 0.7592267135325131, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 52.43232965865677, "median_score_margin": 72.44926834106445, "negative_margin_rate": 0.3245459871118922, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999792588407, "unknown_mean_need_prob": 0.0003750932634526786, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.5046728971962616, "recall1": 0.5046728971962616, "recall3": 0.8785046728971962, "recall5": 0.9719626168224299, "recall8": 1.0, "evidence_count_in_top3": 0.8785046728971962, "hit3": 0.8785046728971962, "mrr": 0.7002447708055184, "ndcg3": 0.7258507766355217, "mean_first_positive_rank": 1.925233644859813, "all_evidence_in_top3": 0.8785046728971962, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 6.655256898882233, "median_score_margin": 1.441606044769287, "negative_margin_rate": 0.4953271028037383, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0003750932634526786, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9945, "recall1": 0.9945, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99725, "ndcg3": 0.9979701136446429, "mean_first_positive_rank": 1.0055, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 243.0819873433709, "median_score_margin": 235.45626258850098, "negative_margin_rate": 0.0055, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9785, "recall1": 0.9785, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98925, "ndcg3": 0.9920649897017864, "mean_first_positive_rank": 1.0215, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 229.82924858646467, "median_score_margin": 248.98048400878906, "negative_margin_rate": 0.0215, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9845, "recall1": 0.9845, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9921666666666666, "ndcg3": 0.9942139463035718, "mean_first_positive_rank": 1.016, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 232.02638172297924, "median_score_margin": 249.2184543609619, "negative_margin_rate": 0.0155, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.2955399044485864e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9815, "recall1": 0.9815, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9906666666666666, "ndcg3": 0.9931067355642863, "mean_first_positive_rank": 1.019, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 211.99539810842276, "median_score_margin": 225.90464401245117, "negative_margin_rate": 0.0185, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.989, "recall1": 0.4945, "recall3": 0.98525, "recall5": 0.996, "recall8": 0.99675, "evidence_count_in_top3": 1.9705, "hit3": 1.0, "mrr": 0.9945, "ndcg3": 0.9737356054745618, "mean_first_positive_rank": 1.011, "all_evidence_in_top3": 0.9705, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9705, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 234.29880876231567, "median_score_margin": 254.67290210723877, "negative_margin_rate": 0.011, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999997019767, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.997, "recall1": 0.997, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9985, "ndcg3": 0.9988927892607143, "mean_first_positive_rank": 1.003, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 155.67033711057155, "median_score_margin": 155.824444770813, "negative_margin_rate": 0.003, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.981, "recall1": 0.981, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9905, "ndcg3": 0.9929876653178578, "mean_first_positive_rank": 1.019, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 217.68418576198818, "median_score_margin": 236.54215621948242, "negative_margin_rate": 0.019, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.969, "recall1": 0.969, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98425, "ndcg3": 0.988362427730358, "mean_first_positive_rank": 1.0325, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 214.68668193136529, "median_score_margin": 221.31105995178223, "negative_margin_rate": 0.031, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.665380374862183, "recall1": 0.665380374862183, "recall3": 0.7662624035281147, "recall5": 0.8280044101433297, "recall8": 0.9178610804851157, "evidence_count_in_top3": 0.7662624035281147, "hit3": 0.7662624035281147, "mrr": 0.745321547104062, "ndcg3": 0.7247713833753366, "mean_first_positive_rank": 2.7304299889746417, "all_evidence_in_top3": 0.7662624035281147, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 49.73213848704934, "median_score_margin": 64.13029479980469, "negative_margin_rate": 0.33461962513781696, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999804822718, "unknown_mean_need_prob": 0.0001509735806525834, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 2.0705206352511507e-15, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "XL-512 v6 best": { "checkpoint": "checkpoints\\router_v6_xl512\\router_best.pt", "router": { "kind": "router_xl", "step": 36000, "arch_config": { "arch": "router_xl", "arch_version": 1, "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3, "encoder_layers": 2, "encoder_hidden": 512, "pair_blocks": 1, "pair_hidden": 512, "pair_expansion": 2, "pair_dropout": 0.05, "use_interaction": true, "policy_layers": 2, "policy_hidden": 512, "policy_dropout": 0.0, "learnable_cosine_scale": true }, "parameters": 7898127, "checkpoint_bytes": 99014505 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048, "checkpoint_bytes": 99014505 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.5855000019655563, "single_query_latency_ms_p50": 1.5855000019655563, "single_query_latency_ms_p95": 1.6239000015048077, "queries_per_second": 630.7158617220388, "batched_qps_64": 39909.70428801726, "batched_qps_256": 94647.25411335309, "batched_latency_ms_64": 0.025056562503777968, "samples": 5, "source": "bench_router_latency.py (interleaved round-robin medians)" }, "cpu": { "single_query_latency_ms_mean": 1.7938500001037028, "single_query_latency_ms_p50": 1.812500002415618, "single_query_latency_ms_p95": 2.0920999995723832, "queries_per_second": 557.4602112451932, "samples": 30, "batched_qps_64": 1675.9990132534788, "batched_latency_ms_64": 0.5966590625007484, "batched_qps_256": 1750.842524579076, "batched_latency_ms_256": 0.5711535937479084 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.957842146626249, "recall1": 0.9024082182553048, "recall3": 0.9758897496351184, "recall5": 0.9834399910183003, "recall8": 0.9905411474121477, "evidence_count_in_top3": 1.0808914337038285, "hit3": 0.9831593128999663, "mrr": 0.9724377931581535, "ndcg3": 0.9664212737428025, "mean_first_positive_rank": 1.1337150555742674, "all_evidence_in_top3": 0.9686201863702706, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.8705, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 45.56406442258217, "median_score_margin": 47.89992094039917, "negative_margin_rate": 0.042157853373750984, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999977515257, "unknown_mean_need_prob": 9.165001448203187e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.130629900134017e-05, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.98375, "recall1": 0.92203125, "recall3": 0.99190625, "recall5": 0.994875, "recall8": 0.99640625, "evidence_count_in_top3": 1.1088125, "hit3": 1.0, "mrr": 0.99184375, "ndcg3": 0.9863265131728122, "mean_first_positive_rank": 1.0164375, "all_evidence_in_top3": 0.9838125, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.8705, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 48.28189839535998, "median_score_margin": 49.04537320137024, "negative_margin_rate": 0.01625, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999552965, "unknown_mean_need_prob": 2.862315262235959e-10, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.7457527826596367, "recall1": 0.7457527826596367, "recall3": 0.8347978910369068, "recall5": 0.8763913298183948, "recall8": 0.9355594610427651, "evidence_count_in_top3": 0.8347978910369068, "hit3": 0.8347978910369068, "mrr": 0.8100220298155272, "ndcg3": 0.7979455040170743, "mean_first_positive_rank": 2.16813122437024, "all_evidence_in_top3": 0.8347978910369068, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 22.8850553075215, "median_score_margin": 27.899999618530273, "negative_margin_rate": 0.2542472173403632, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999769542673, "unknown_mean_need_prob": 0.0008787320924793242, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.4672897196261682, "recall1": 0.4672897196261682, "recall3": 0.8317757009345794, "recall5": 0.9813084112149533, "recall8": 0.9906542056074766, "evidence_count_in_top3": 0.8317757009345794, "hit3": 0.8317757009345794, "mrr": 0.6616751628433871, "ndcg3": 0.6776764890854534, "mean_first_positive_rank": 2.1682242990654204, "all_evidence_in_top3": 0.8317757009345794, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.9633634409336286, "median_score_margin": -1.062392234802246, "negative_margin_rate": 0.5327102803738317, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0008787320924793242, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.992, "recall1": 0.992, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.996, "ndcg3": 0.9970474380285717, "mean_first_positive_rank": 1.008, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 67.69295654568822, "median_score_margin": 69.3216142654419, "negative_margin_rate": 0.008, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9795, "recall1": 0.9795, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98975, "ndcg3": 0.9924340599482149, "mean_first_positive_rank": 1.0205, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 39.98083441020921, "median_score_margin": 44.252100229263306, "negative_margin_rate": 0.0205, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9805, "recall1": 0.9805, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99025, "ndcg3": 0.9928031301946435, "mean_first_positive_rank": 1.0195, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 40.70788796337694, "median_score_margin": 44.594672441482544, "negative_margin_rate": 0.0195, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 5.724454182644648e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9815, "recall1": 0.9815, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9906666666666666, "ndcg3": 0.9931067355642863, "mean_first_positive_rank": 1.019, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 37.93500039816182, "median_score_margin": 40.258564949035645, "negative_margin_rate": 0.0185, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9875, "recall1": 0.49375, "recall3": 0.93525, "recall5": 0.959, "recall8": 0.97125, "evidence_count_in_top3": 1.8705, "hit3": 1.0, "mrr": 0.9936666666666666, "ndcg3": 0.9341087890914226, "mean_first_positive_rank": 1.013, "all_evidence_in_top3": 0.8705, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.8705, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 47.13378771021217, "median_score_margin": 51.072168588638306, "negative_margin_rate": 0.0125, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999996423722, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9965, "recall1": 0.9965, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99825, "ndcg3": 0.9987082541375002, "mean_first_positive_rank": 1.0035, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 61.5167154010348, "median_score_margin": 62.94392251968384, "negative_margin_rate": 0.0035, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9815, "recall1": 0.9815, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99075, "ndcg3": 0.9931722004410719, "mean_first_positive_rank": 1.0185, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 38.85421975574363, "median_score_margin": 43.87380111217499, "negative_margin_rate": 0.0185, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.971, "recall1": 0.971, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9854166666666666, "ndcg3": 0.9892314979767864, "mean_first_positive_rank": 1.0295, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 52.43378497845307, "median_score_margin": 59.86355948448181, "negative_margin_rate": 0.029, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.7293274531422271, "recall1": 0.7293274531422271, "recall3": 0.834619625137817, "recall5": 0.88257993384785, "recall8": 0.9388092613009923, "evidence_count_in_top3": 0.834619625137817, "hit3": 0.834619625137817, "mrr": 0.8012716909147449, "ndcg3": 0.7908513559477891, "mean_first_positive_rank": 2.1681367144432193, "all_evidence_in_top3": 0.834619625137817, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 21.591989690253087, "median_score_margin": 24.97784423828125, "negative_margin_rate": 0.27067254685777287, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999783136353, "unknown_mean_need_prob": 0.0003550033115114848, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.7634182726834057e-14, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "V2-512 v6 best": { "checkpoint": "checkpoints\\router_v6_v2_512\\router_best.pt", "router": { "kind": "router_v2", "step": 2000, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3 }, "parameters": 4741902, "checkpoint_bytes": 56922573 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048, "checkpoint_bytes": 56922573 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.162250000561471, "single_query_latency_ms_p50": 1.162250000561471, "single_query_latency_ms_p95": 1.1778999996749917, "queries_per_second": 860.4000856243581, "batched_qps_64": 53757.11860745415, "batched_qps_256": 154085.05975293368, "batched_latency_ms_64": 0.01860218750380227, "samples": 5, "source": "bench_router_latency.py (interleaved round-robin medians)" }, "cpu": { "single_query_latency_ms_mean": 0.8890533339581452, "single_query_latency_ms_p50": 0.8698999990883749, "single_query_latency_ms_p95": 1.0765000006358605, "queries_per_second": 1124.7919127055181, "samples": 30, "batched_qps_64": 3131.6700902508155, "batched_latency_ms_64": 0.31931843750498956, "batched_qps_256": 3247.1683677121205, "batched_latency_ms_256": 0.30796062499973686 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.950488379925901, "recall1": 0.8971034018187942, "recall3": 0.9884079937128102, "recall5": 0.9934882676546536, "recall8": 0.9961266419669923, "evidence_count_in_top3": 1.0986302907825305, "hit3": 0.9904569439766476, "mrr": 0.9711571604054701, "ndcg3": 0.9723689530227545, "mean_first_positive_rank": 1.1156393847535646, "all_evidence_in_top3": 0.9863590434489727, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9635, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 12.36926158705623, "median_score_margin": 12.002254128456116, "negative_margin_rate": 0.049511620074099025, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9988408471058552, "unknown_mean_need_prob": 0.006478862109410833, "thr0.30": { "need_precision": 0.999326825984517, "need_recall": 1.0, "need_f1": 0.9996632996632996, "specificity_unknown_refusal": 0.9970774476376035, "abstention_accuracy": 0.9994525547445255, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0029225523623964927, "counts": { "tp": 17814, "tn": 4094, "fp": 12, "fn": 0 } }, "thr0.40": { "need_precision": 0.999326825984517, "need_recall": 1.0, "need_f1": 0.9996632996632996, "specificity_unknown_refusal": 0.9970774476376035, "abstention_accuracy": 0.9994525547445255, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0029225523623964927, "counts": { "tp": 17814, "tn": 4094, "fp": 12, "fn": 0 } }, "thr0.50": { "need_precision": 0.9995511165974638, "need_recall": 1.0, "need_f1": 0.999775507913346, "specificity_unknown_refusal": 0.9980516317584024, "abstention_accuracy": 0.9996350364963503, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.001948368241597662, "counts": { "tp": 17814, "tn": 4098, "fp": 8, "fn": 0 } }, "thr0.60": { "need_precision": 0.9998877413560844, "need_recall": 1.0, "need_f1": 0.9999438675273646, "specificity_unknown_refusal": 0.9995129079396006, "abstention_accuracy": 0.9999087591240876, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0004870920603994155, "counts": { "tp": 17814, "tn": 4104, "fp": 2, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.0904978112084791, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.97, "recall1": 0.9105625, "recall3": 0.99771875, "recall5": 0.9999375, "recall8": 1.0, "evidence_count_in_top3": 1.1204375, "hit3": 1.0, "mrr": 0.9849166666666667, "ndcg3": 0.9857856193167982, "mean_first_positive_rank": 1.0305, "all_evidence_in_top3": 0.9954375, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9635, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 13.542529965203517, "median_score_margin": 12.727105312049389, "negative_margin_rate": 0.03, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9990760059095919, "unknown_mean_need_prob": 0.0025661538663334796, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.8125366139425894, "recall1": 0.8125366139425894, "recall3": 0.9244288224956063, "recall5": 0.9502050380785003, "recall8": 0.971294669009959, "evidence_count_in_top3": 0.9244288224956063, "hit3": 0.9244288224956063, "mrr": 0.8742941912651278, "ndcg3": 0.878837443896981, "mean_first_positive_rank": 1.7176332747510252, "all_evidence_in_top3": 0.9244288224956063, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.2332465364638656, "median_score_margin": 2.495278835296631, "negative_margin_rate": 0.18746338605741067, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.996912036259131, "unknown_mean_need_prob": 0.2510888675848643, "thr0.50": { "need_precision": 0.9976621858562245, "need_recall": 1.0, "need_f1": 0.9988297249853716, "specificity_unknown_refusal": 0.8095238095238095, "abstention_accuracy": 0.9976851851851852, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.19047619047619047, "counts": { "tp": 1707, "tn": 17, "fp": 4, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.2336448598130841, "recall1": 0.2336448598130841, "recall3": 0.616822429906542, "recall5": 0.719626168224299, "recall8": 0.8130841121495327, "evidence_count_in_top3": 0.616822429906542, "hit3": 0.616822429906542, "mrr": 0.4589420963252739, "ndcg3": 0.4582719938918631, "mean_first_positive_rank": 4.242990654205608, "all_evidence_in_top3": 0.616822429906542, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -1.3701436372003823, "median_score_margin": -0.6190404891967773, "negative_margin_rate": 0.7663551401869159, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 0.9944477560364198, "unknown_mean_need_prob": 0.2510888675848643, "thr0.50": { "need_precision": 0.963963963963964, "need_recall": 1.0, "need_f1": 0.9816513761467891, "specificity_unknown_refusal": 0.8095238095238095, "abstention_accuracy": 0.96875, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.19047619047619047, "counts": { "tp": 107, "tn": 17, "fp": 4, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9905, "recall1": 0.9905, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99525, "ndcg3": 0.9964938326589288, "mean_first_positive_rank": 1.0095, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 16.784344019554556, "median_score_margin": 14.332054734230042, "negative_margin_rate": 0.0095, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9984212898612023, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.974, "recall1": 0.974, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.987, "ndcg3": 0.9904041735928579, "mean_first_positive_rank": 1.026, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 14.32004562981613, "median_score_margin": 12.83707270026207, "negative_margin_rate": 0.026, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9998619016408921, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.969, "recall1": 0.969, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9844166666666666, "ndcg3": 0.9884933574839294, "mean_first_positive_rank": 1.0315, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 14.515158921549562, "median_score_margin": 12.90774929523468, "negative_margin_rate": 0.031, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9998633873462677, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.004985218498797622, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.973, "recall1": 0.973, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9864166666666666, "ndcg3": 0.9899696384696437, "mean_first_positive_rank": 1.0275, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 11.82059130709432, "median_score_margin": 12.446841090917587, "negative_margin_rate": 0.027, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9998804149031639, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.951, "recall1": 0.4755, "recall3": 0.98175, "recall5": 0.9995, "recall8": 1.0, "evidence_count_in_top3": 1.9635, "hit3": 1.0, "mrr": 0.9753333333333334, "ndcg3": 0.9571701608629515, "mean_first_positive_rank": 1.05, "all_evidence_in_top3": 0.9635, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9635, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 10.0001081677787, "median_score_margin": 11.238629460334778, "negative_margin_rate": 0.049, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9978087923228741, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9755, "recall1": 0.9755, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98775, "ndcg3": 0.9909577789625007, "mean_first_positive_rank": 1.0245, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 14.530858453191817, "median_score_margin": 12.788989305496216, "negative_margin_rate": 0.0245, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9972889910042286, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9745, "recall1": 0.9745, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9871666666666666, "ndcg3": 0.9905232438392865, "mean_first_positive_rank": 1.026, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 13.56638970718137, "median_score_margin": 13.154640913009644, "negative_margin_rate": 0.0255, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9995595035254955, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9525, "recall1": 0.9525, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.976, "ndcg3": 0.982272768664287, "mean_first_positive_rank": 1.049, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 12.802743515461684, "median_score_margin": 12.365995168685913, "negative_margin_rate": 0.0475, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999237666726113, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.7783902976846747, "recall1": 0.7783902976846747, "recall3": 0.906284454244763, "recall5": 0.9366041896361632, "recall8": 0.961962513781698, "evidence_count_in_top3": 0.906284454244763, "hit3": 0.906284454244763, "mrr": 0.8497943708910569, "ndcg3": 0.8540301102968997, "mean_first_positive_rank": 1.866593164277839, "all_evidence_in_top3": 0.906284454244763, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.0206981634858754, "median_score_margin": 2.3058557510375977, "negative_margin_rate": 0.22160970231532526, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9967666790464352, "unknown_mean_need_prob": 0.1541282297727072, "thr0.50": { "need_precision": 0.995609220636663, "need_recall": 1.0, "need_f1": 0.9977997799779977, "specificity_unknown_refusal": 0.9245283018867925, "abstention_accuracy": 0.9958333333333333, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.07547169811320754, "counts": { "tp": 1814, "tn": 98, "fp": 8, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.00014708923386933748, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } } }