{ "V2-128 deployed(v3)": { "checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_qwen_router_entities\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 8157561 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 8157561 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.8439246668907193, "single_query_latency_ms_p50": 0.8266000004368834, "single_query_latency_ms_p95": 1.1234000012336764, "queries_per_second": 1184.9398876848934, "samples": 150, "batched_qps_64": 72916.19191648123, "batched_latency_ms_64": 0.013714375006657065, "batched_qps_256": 264812.97583671653, "batched_latency_ms_256": 0.003776249999987158 }, "cpu": { "single_query_latency_ms_mean": 0.3935313334174377, "single_query_latency_ms_p50": 0.3775000004679896, "single_query_latency_ms_p95": 0.5684999996447004, "queries_per_second": 2541.093720075529, "samples": 150, "batched_qps_64": 11881.908679212764, "batched_latency_ms_64": 0.08416156250632412, "batched_qps_256": 13117.630300343435, "batched_latency_ms_256": 0.07623328124850559 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.4112495789828225, "recall1": 0.37027057370607386, "recall3": 0.4673009992141013, "recall5": 0.4891377568204783, "recall8": 0.5160267205568654, "evidence_count_in_top3": 0.5420455821264174, "hit3": 0.5003368137420007, "mrr": 0.476908110965675, "ndcg3": 0.44220238482445295, "mean_first_positive_rank": 14.38671831144044, "all_evidence_in_top3": 0.43426518468620184, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.3715, "hop_accuracy": 0.7322992700729927, "hop_accuracy_answerable": 0.9010890310991355, "hop_under_prediction": 0.046817110138093636, "hop_over_prediction": 0.05209385876277085, "mean_score_margin": -9.29892828391031, "median_score_margin": -9.269989609718323, "negative_margin_rate": 0.5887504210171774, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 4042, "true_0_pred_2": 64, "true_1_pred_0": 834, "true_1_pred_1": 14052, "true_1_pred_2": 928, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9762653562039442, "unknown_mean_need_prob": 0.998855464207621, "thr0.30": { "need_precision": 0.8126824817518248, "need_recall": 1.0, "need_f1": 0.8966628076710123, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8126824817518248, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 17814, "tn": 0, "fp": 4106, "fn": 0 } }, "thr0.40": { "need_precision": 0.8126824817518248, "need_recall": 1.0, "need_f1": 0.8966628076710123, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8126824817518248, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 17814, "tn": 0, "fp": 4106, "fn": 0 } }, "thr0.50": { "need_precision": 0.8124086257309941, "need_recall": 0.998203660042663, "need_f1": 0.895773512669387, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8112226277372263, "known_question_refusal_rate": 0.001796339957336926, "unknown_question_read_rate": 1.0, "counts": { "tp": 17782, "tn": 0, "fp": 4106, "fn": 32 } }, "thr0.60": { "need_precision": 0.8115476409032495, "need_recall": 0.9925900976759852, "need_f1": 0.892985202767537, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8066605839416059, "known_question_refusal_rate": 0.00740990232401482, "unknown_question_read_rate": 1.0, "counts": { "tp": 17682, "tn": 0, "fp": 4106, "fn": 132 } }, "thr0.70": { "need_precision": 0.8112270700197692, "need_recall": 0.9905130796003143, "need_f1": 0.8919499557689878, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8049726277372263, "known_question_refusal_rate": 0.00948692039968564, "unknown_question_read_rate": 1.0, "counts": { "tp": 17645, "tn": 0, "fp": 4106, "fn": 169 } }, "thr0.80": { "need_precision": 0.8106525247867189, "need_recall": 0.9868081284383069, "need_f1": 0.8900984835059115, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8019616788321168, "known_question_refusal_rate": 0.01319187156169305, "unknown_question_read_rate": 1.0, "counts": { "tp": 17579, "tn": 0, "fp": 4106, "fn": 235 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_2": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.9999341862276196, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 1.0, "counts": { "tp": 0, "tn": 0, "fp": 64, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.442375, "recall1": 0.39675, "recall3": 0.49153125, "recall5": 0.50303125, "recall8": 0.51234375, "evidence_count_in_top3": 0.57475, "hit3": 0.5283125, "mrr": 0.500402578720693, "ndcg3": 0.4692792057097287, "mean_first_positive_rank": 14.9525, "all_evidence_in_top3": 0.45475, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.3715, "hop_accuracy": 0.74955, "hop_accuracy_answerable": 0.9369375, "hop_under_prediction": 0.0160625, "hop_over_prediction": 0.047, "mean_score_margin": -8.091850390457548, "median_score_margin": -6.662378132343292, "negative_margin_rate": 0.557625, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 4000, "true_0_pred_2": 0, "true_1_pred_0": 257, "true_1_pred_1": 12991, "true_1_pred_2": 752, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9821553228273988, "unknown_mean_need_prob": 0.9988265322595835, "thr0.50": { "need_precision": 0.8, "need_recall": 1.0, "need_f1": 0.888888888888889, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 16000, "tn": 0, "fp": 4000, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.14528412419449327, "recall1": 0.14528412419449327, "recall3": 0.2677211482132396, "recall5": 0.38488576449912126, "recall8": 0.5653192735793791, "evidence_count_in_top3": 0.2677211482132396, "hit3": 0.2677211482132396, "mrr": 0.2801716446629072, "ndcg3": 0.21516992510461316, "mean_first_positive_rank": 9.195664909197422, "all_evidence_in_top3": 0.2677211482132396, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.5740740740740741, "hop_accuracy_answerable": 0.5811364967779731, "hop_under_prediction": 0.3380199179847686, "hop_over_prediction": 0.08084358523725835, "mean_score_margin": -19.359866383266713, "median_score_margin": -18.159542322158813, "negative_margin_rate": 0.8547158758055068, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 21, "true_0_pred_2": 0, "true_1_pred_0": 577, "true_1_pred_1": 992, "true_1_pred_2": 138 }, "known_mean_need_prob": 0.9195722371566764, "unknown_mean_need_prob": 0.9999671209426153, "thr0.50": { "need_precision": 0.9876179245283019, "need_recall": 0.981253661394259, "need_f1": 0.9844255069056715, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.9693287037037037, "known_question_refusal_rate": 0.018746338605741066, "unknown_question_read_rate": 1.0, "counts": { "tp": 1675, "tn": 0, "fp": 21, "fn": 32 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.0, "recall1": 0.0, "recall3": 0.028037383177570093, "recall5": 0.07476635514018691, "recall8": 0.2803738317757009, "evidence_count_in_top3": 0.028037383177570093, "hit3": 0.028037383177570093, "mrr": 0.1023068389893556, "ndcg3": 0.01524233414552764, "mean_first_positive_rank": 12.598130841121495, "all_evidence_in_top3": 0.028037383177570093, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.5390625, "hop_accuracy_answerable": 0.6448598130841121, "hop_under_prediction": 0.0, "hop_over_prediction": 0.35514018691588783, "mean_score_margin": -29.291684916086286, "median_score_margin": -29.07153034210205, "negative_margin_rate": 1.0, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 21, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 69, "true_1_pred_2": 38 }, "known_mean_need_prob": 0.9999633771236812, "unknown_mean_need_prob": 0.9999671209426153, "thr0.50": { "need_precision": 0.8359375, "need_recall": 1.0, "need_f1": 0.9106382978723404, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8359375, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 107, "tn": 0, "fp": 21, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.796, "recall1": 0.796, "recall3": 0.9125, "recall5": 0.919, "recall8": 0.927, "evidence_count_in_top3": 0.9125, "hit3": 0.9125, "mrr": 0.8582544951434297, "ndcg3": 0.8678012294946459, "mean_first_positive_rank": 2.8275, "all_evidence_in_top3": 0.9125, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.4955, "hop_accuracy_answerable": 0.4955, "hop_under_prediction": 0.1285, "hop_over_prediction": 0.376, "mean_score_margin": 21.036133953779935, "median_score_margin": 21.400100469589233, "negative_margin_rate": 0.204, "hop_confusion": { "true_1_pred_0": 257, "true_1_pred_1": 991, "true_1_pred_2": 752 }, "known_mean_need_prob": 0.9764632563889026, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.231, "recall1": 0.231, "recall3": 0.2535, "recall5": 0.255, "recall8": 0.257, "evidence_count_in_top3": 0.2535, "hit3": 0.2535, "mrr": 0.26603145346142226, "ndcg3": 0.24486859507142916, "mean_first_positive_rank": 23.789, "all_evidence_in_top3": 0.2535, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -27.718347060799598, "median_score_margin": -38.59446668624878, "negative_margin_rate": 0.769, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9723067943453789, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.2255, "recall1": 0.2255, "recall3": 0.2455, "recall5": 0.2475, "recall8": 0.2485, "evidence_count_in_top3": 0.2455, "hit3": 0.2455, "mrr": 0.2596338049221357, "ndcg3": 0.23792220044107196, "mean_first_positive_rank": 24.0695, "all_evidence_in_top3": 0.2455, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -28.903205164782705, "median_score_margin": -39.01692509651184, "negative_margin_rate": 0.7745, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9724746745526791, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.99774328109622, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 1.0, "counts": { "tp": 0, "tn": 0, "fp": 2000, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.5255, "recall1": 0.5255, "recall3": 0.6245, "recall5": 0.6265, "recall8": 0.6295, "evidence_count_in_top3": 0.6245, "hit3": 0.6245, "mrr": 0.5855216777342396, "ndcg3": 0.585867169546431, "mean_first_positive_rank": 12.2085, "all_evidence_in_top3": 0.6245, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -1.0039909677058458, "median_score_margin": 2.929476022720337, "negative_margin_rate": 0.4745, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9875293919146061, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.73, "recall1": 0.365, "recall3": 0.66575, "recall5": 0.72725, "recall8": 0.77375, "evidence_count_in_top3": 1.3315, "hit3": 0.96, "mrr": 0.84532448592014, "ndcg3": 0.6658792959224611, "mean_first_positive_rank": 1.59, "all_evidence_in_top3": 0.3715, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.3715, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.786984693795443, "median_score_margin": 5.27826452255249, "negative_margin_rate": 0.27, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9993210740983486, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.443, "recall1": 0.443, "recall3": 0.5525, "recall5": 0.5615, "recall8": 0.5665, "evidence_count_in_top3": 0.5525, "hit3": 0.5525, "mrr": 0.5118924660255056, "ndcg3": 0.5092063534375025, "mean_first_positive_rank": 13.8255, "all_evidence_in_top3": 0.5525, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -2.995655853047967, "median_score_margin": -5.5251545906066895, "negative_margin_rate": 0.557, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9880377069413662, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.249, "recall1": 0.249, "recall3": 0.287, "recall5": 0.293, "recall8": 0.3005, "evidence_count_in_top3": 0.287, "hit3": 0.287, "mrr": 0.2919631936667751, "ndcg3": 0.27173149797678653, "mean_first_positive_rank": 22.067, "all_evidence_in_top3": 0.287, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -19.72415616109967, "median_score_margin": -21.928287506103516, "negative_margin_rate": 0.751, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9656237106621265, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.339, "recall1": 0.339, "recall3": 0.391, "recall5": 0.3945, "recall8": 0.396, "evidence_count_in_top3": 0.391, "hit3": 0.391, "mrr": 0.3845990528918967, "ndcg3": 0.3709573037875013, "mean_first_positive_rank": 19.243, "all_evidence_in_top3": 0.391, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -8.212566563799978, "median_score_margin": -16.937832549214363, "negative_margin_rate": 0.661, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9954859737157822, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.13671444321940462, "recall1": 0.13671444321940462, "recall3": 0.2535832414553473, "recall5": 0.36659316427783906, "recall8": 0.5485115766262404, "evidence_count_in_top3": 0.2535832414553473, "hit3": 0.2535832414553473, "mrr": 0.26968017045834825, "ndcg3": 0.2033770627933551, "mean_first_positive_rank": 9.396361631753033, "all_evidence_in_top3": 0.2535832414553473, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.5526041666666667, "hop_accuracy_answerable": 0.5848952590959207, "hop_under_prediction": 0.31808158765159866, "hop_over_prediction": 0.0970231532524807, "mean_score_margin": -19.945701324287494, "median_score_margin": -18.862584590911865, "negative_margin_rate": 0.8632855567805954, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 42, "true_0_pred_2": 64, "true_1_pred_0": 577, "true_1_pred_1": 1061, "true_1_pred_2": 176 }, "known_mean_need_prob": 0.9243141621712682, "unknown_mean_need_prob": 0.9999472358316746, "thr0.50": { "need_precision": 0.9438559322033898, "need_recall": 0.9823594266813671, "need_f1": 0.9627228525121555, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.928125, "known_question_refusal_rate": 0.017640573318632856, "unknown_question_read_rate": 1.0, "counts": { "tp": 1782, "tn": 0, "fp": 106, "fn": 32 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.9999097834229469, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 1.0, "counts": { "tp": 0, "tn": 0, "fp": 2000, "fn": 0 } } } } }, "V2-512 v3 best": { "checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_router_512\\router_best.pt", "router": { "kind": "router_v2", "step": 1000, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3 }, "parameters": 4741902, "checkpoint_bytes": 56922381 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048, "checkpoint_bytes": 56922381 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.8789160000742413, "single_query_latency_ms_p50": 0.7994000006874558, "single_query_latency_ms_p95": 1.517499997135019, "queries_per_second": 1137.7651560735392, "samples": 150, "batched_qps_64": 69808.02789170416, "batched_latency_ms_64": 0.014325000006465416, "batched_qps_256": 167235.0043735693, "batched_latency_ms_256": 0.005979609375117434 }, "cpu": { "single_query_latency_ms_mean": 0.8516033333095644, "single_query_latency_ms_p50": 0.8521000017935876, "single_query_latency_ms_p95": 1.0607999975036364, "queries_per_second": 1174.2556198244615, "samples": 150, "batched_qps_64": 3215.094870427158, "batched_latency_ms_64": 0.3110328124989792, "batched_qps_256": 3174.0694397097395, "batched_latency_ms_256": 0.31505296875025124 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.807510946446615, "recall1": 0.7698439429662064, "recall3": 0.9501515661839003, "recall5": 0.9730829684517796, "recall8": 0.984506567867969, "evidence_count_in_top3": 1.052542943752105, "hit3": 0.9584035028629168, "mrr": 0.8845870538857731, "ndcg3": 0.8953026178367461, "mean_first_positive_rank": 1.6810935219490288, "all_evidence_in_top3": 0.9418996295048838, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.8385, "hop_accuracy": 0.8158759124087591, "hop_accuracy_answerable": 0.8877287526664421, "hop_under_prediction": 0.11227124733355788, "hop_over_prediction": 0.0, "mean_score_margin": 9.638220129697089, "median_score_margin": 7.411673903465271, "negative_margin_rate": 0.19248905355338497, "hop_confusion": { "true_0_pred_0": 2070, "true_0_pred_1": 2036, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 2000, "true_2_pred_1": 0, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.8877228112198915, "unknown_mean_need_prob": 0.6232877522355414, "thr0.30": { "need_precision": 0.8045772861057546, "need_recall": 0.8900303132367801, "need_f1": 0.8451492537313433, "specificity_unknown_refusal": 0.06210423770092548, "abstention_accuracy": 0.7349452554744526, "known_question_refusal_rate": 0.10996968676321994, "unknown_question_read_rate": 0.9378957622990746, "counts": { "tp": 15855, "tn": 255, "fp": 3851, "fn": 1959 } }, "thr0.40": { "need_precision": 0.885845516159749, "need_recall": 0.8877848882901089, "need_f1": 0.8868141419239074, "specificity_unknown_refusal": 0.5036531904529956, "abstention_accuracy": 0.8158302919708029, "known_question_refusal_rate": 0.1122151117098911, "unknown_question_read_rate": 0.4963468095470044, "counts": { "tp": 15815, "tn": 2068, "fp": 2038, "fn": 1999 } }, "thr0.50": { "need_precision": 0.8859383753501401, "need_recall": 0.8877287526664421, "need_f1": 0.8868326603858232, "specificity_unknown_refusal": 0.504140282513395, "abstention_accuracy": 0.8158759124087591, "known_question_refusal_rate": 0.11227124733355788, "unknown_question_read_rate": 0.49585971748660496, "counts": { "tp": 15814, "tn": 2070, "fp": 2036, "fn": 2000 } }, "thr0.60": { "need_precision": 0.8868326603858232, "need_recall": 0.8877287526664421, "need_f1": 0.8872804802782921, "specificity_unknown_refusal": 0.5085241110569898, "abstention_accuracy": 0.8166970802919709, "known_question_refusal_rate": 0.11227124733355788, "unknown_question_read_rate": 0.49147588894301025, "counts": { "tp": 15814, "tn": 2088, "fp": 2018, "fn": 2000 } }, "thr0.70": { "need_precision": 0.8876290974405029, "need_recall": 0.8877287526664421, "need_f1": 0.8876789222565254, "specificity_unknown_refusal": 0.5124208475401851, "abstention_accuracy": 0.8174270072992701, "known_question_refusal_rate": 0.11227124733355788, "unknown_question_read_rate": 0.4875791524598149, "counts": { "tp": 15814, "tn": 2104, "fp": 2002, "fn": 2000 } }, "thr0.80": { "need_precision": 0.9022136010953903, "need_recall": 0.8877287526664421, "need_f1": 0.8949125686152454, "specificity_unknown_refusal": 0.5825621042377009, "abstention_accuracy": 0.8305656934306569, "known_question_refusal_rate": 0.11227124733355788, "unknown_question_read_rate": 0.41743789576229906, "counts": { "tp": 15814, "tn": 2392, "fp": 1714, "fn": 2000 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.04609098758373875, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.8444375, "recall1": 0.8025, "recall3": 0.9736875, "recall5": 0.98640625, "recall8": 0.986875, "evidence_count_in_top3": 1.0876875, "hit3": 0.982875, "mrr": 0.9118529559094745, "ndcg3": 0.9247008759665758, "mean_first_positive_rank": 1.5631875, "all_evidence_in_top3": 0.9645, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.8385, "hop_accuracy": 0.8, "hop_accuracy_answerable": 0.875, "hop_under_prediction": 0.125, "hop_over_prediction": 0.0, "mean_score_margin": 10.705225070305634, "median_score_margin": 8.861685633659363, "negative_margin_rate": 0.1555625, "hop_confusion": { "true_0_pred_0": 2000, "true_0_pred_1": 2000, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 2000, "true_2_pred_1": 0, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.8765438592466526, "unknown_mean_need_prob": 0.6330925791561604, "thr0.50": { "need_precision": 0.875, "need_recall": 0.875, "need_f1": 0.875, "specificity_unknown_refusal": 0.5, "abstention_accuracy": 0.8, "known_question_refusal_rate": 0.125, "unknown_question_read_rate": 0.5, "counts": { "tp": 14000, "tn": 2000, "fp": 2000, "fn": 2000 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.4856473345049795, "recall1": 0.4856473345049795, "recall3": 0.7375512595196251, "recall5": 0.8482718219097832, "recall8": 0.961335676625659, "evidence_count_in_top3": 0.7375512595196251, "hit3": 0.7375512595196251, "mrr": 0.6442412517298798, "ndcg3": 0.6345330968470215, "mean_first_positive_rank": 2.754540128881078, "all_evidence_in_top3": 0.7375512595196251, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.9895833333333334, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.24317214623591182, "median_score_margin": -0.0259857177734375, "negative_margin_rate": 0.5143526654950205, "hop_confusion": { "true_0_pred_0": 3, "true_0_pred_1": 18, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9860711831339773, "unknown_mean_need_prob": 0.5690326392650604, "thr0.50": { "need_precision": 0.9895652173913043, "need_recall": 1.0, "need_f1": 0.9947552447552448, "specificity_unknown_refusal": 0.14285714285714285, "abstention_accuracy": 0.9895833333333334, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.8571428571428571, "counts": { "tp": 1707, "tn": 3, "fp": 18, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.4205607476635514, "recall1": 0.4205607476635514, "recall3": 0.822429906542056, "recall5": 0.9719626168224299, "recall8": 1.0, "evidence_count_in_top3": 0.822429906542056, "hit3": 0.822429906542056, "mrr": 0.6417445482866044, "ndcg3": 0.6594282463618242, "mean_first_positive_rank": 2.1869158878504673, "all_evidence_in_top3": 0.822429906542056, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.859375, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.03217372047566922, "median_score_margin": -0.08555316925048828, "negative_margin_rate": 0.5794392523364486, "hop_confusion": { "true_0_pred_0": 3, "true_0_pred_1": 18, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 0.9903635655608133, "unknown_mean_need_prob": 0.5690326392650604, "thr0.50": { "need_precision": 0.856, "need_recall": 1.0, "need_f1": 0.9224137931034483, "specificity_unknown_refusal": 0.14285714285714285, "abstention_accuracy": 0.859375, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.8571428571428571, "counts": { "tp": 107, "tn": 3, "fp": 18, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.8605, "recall1": 0.8605, "recall3": 0.9995, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 0.9995, "hit3": 0.9995, "mrr": 0.9282083333333334, "ndcg3": 0.9466935435803608, "mean_first_positive_rank": 1.152, "all_evidence_in_top3": 0.9995, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 11.369050871308893, "median_score_margin": 8.467366218566895, "negative_margin_rate": 0.1395, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9782380353510379, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9215, "recall1": 0.9215, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9599166666666666, "ndcg3": 0.970373336887502, "mean_first_positive_rank": 1.0835, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 12.625836706157774, "median_score_margin": 9.696466445922852, "negative_margin_rate": 0.0785, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.959310285449028, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9215, "recall1": 0.9215, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.95975, "ndcg3": 0.9702424071339306, "mean_first_positive_rank": 1.0845, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 12.816123366106302, "median_score_margin": 9.613193988800049, "negative_margin_rate": 0.0785, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.959583455324173, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.951540339499712, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 1.0, "counts": { "tp": 0, "tn": 0, "fp": 2000, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.782, "recall1": 0.782, "recall3": 0.989, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 0.989, "hit3": 0.989, "mrr": 0.8819583333333334, "ndcg3": 0.9077580581071478, "mean_first_positive_rank": 1.2795, "all_evidence_in_top3": 0.989, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 12.019205482423306, "median_score_margin": 8.422776699066162, "negative_margin_rate": 0.218, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9750196565389633, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.671, "recall1": 0.3355, "recall3": 0.912, "recall5": 0.99775, "recall8": 1.0, "evidence_count_in_top3": 1.824, "hit3": 0.9855, "mrr": 0.8213583333333333, "ndcg3": 0.8246288558486577, "mean_first_positive_rank": 1.4225, "all_evidence_in_top3": 0.8385, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.8385, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": 5.122980190102011, "median_score_margin": 3.771735429763794, "negative_margin_rate": 0.329, "hop_confusion": { "true_2_pred_0": 2000, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.1813941482268274, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": null, "need_recall": 0.0, "need_f1": null, "specificity_unknown_refusal": null, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": null, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2000 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.7725, "recall1": 0.7725, "recall3": 0.8895, "recall5": 0.8935, "recall8": 0.895, "evidence_count_in_top3": 0.8895, "hit3": 0.8895, "mrr": 0.8329236472757954, "ndcg3": 0.8441584402339315, "mean_first_positive_rank": 4.2945, "all_evidence_in_top3": 0.8895, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 6.7703079060800375, "median_score_margin": 4.96909373998642, "negative_margin_rate": 0.2275, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9814486785233021, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.942, "recall1": 0.942, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.97025, "ndcg3": 0.978004741816073, "mean_first_positive_rank": 1.0625, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 14.161037405148148, "median_score_margin": 10.101282358169556, "negative_margin_rate": 0.058, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9777550499141217, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.8845, "recall1": 0.8845, "recall3": 0.9995, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 0.9995, "hit3": 0.9995, "mrr": 0.9404583333333334, "ndcg3": 0.9557476241250031, "mean_first_positive_rank": 1.1265, "all_evidence_in_top3": 0.9995, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 10.757258635118603, "median_score_margin": 8.949375867843628, "negative_margin_rate": 0.1155, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9996015646457672, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.48180815876515987, "recall1": 0.48180815876515987, "recall3": 0.7425578831312017, "recall5": 0.8555678059536935, "recall8": 0.9636163175303197, "evidence_count_in_top3": 0.7425578831312017, "hit3": 0.7425578831312017, "mrr": 0.6440939820118917, "ndcg3": 0.6360015538470678, "mean_first_positive_rank": 2.7210584343991178, "all_evidence_in_top3": 0.7425578831312017, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.98125, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.2269306866228252, "median_score_margin": -0.02995920181274414, "negative_margin_rate": 0.5181918412348401, "hop_confusion": { "true_0_pred_0": 70, "true_0_pred_1": 36, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.98632437217459, "unknown_mean_need_prob": 0.2532942835329417, "thr0.50": { "need_precision": 0.9805405405405405, "need_recall": 1.0, "need_f1": 0.990174672489083, "specificity_unknown_refusal": 0.660377358490566, "abstention_accuracy": 0.98125, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.33962264150943394, "counts": { "tp": 1814, "tn": 70, "fp": 36, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.3146448188126087, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "V2-512 v3 final": { "checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_router_512\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3 }, "parameters": 4741902, "checkpoint_bytes": 18973433 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048, "checkpoint_bytes": 18973433 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.9094866666661497, "single_query_latency_ms_p50": 0.8472000008623581, "single_query_latency_ms_p95": 1.27029999930528, "queries_per_second": 1099.5213417098676, "samples": 150, "batched_qps_64": 67135.21454097527, "batched_latency_ms_64": 0.014895312494900281, "batched_qps_256": 168449.866383823, "batched_latency_ms_256": 0.005936484376434237 }, "cpu": { "single_query_latency_ms_mean": 0.8548113333866544, "single_query_latency_ms_p50": 0.8628999967186246, "single_query_latency_ms_p95": 1.0559000002103858, "queries_per_second": 1169.8487852730339, "samples": 150, "batched_qps_64": 2969.669466452853, "batched_latency_ms_64": 0.33673781250627144, "batched_qps_256": 3091.357337730013, "batched_latency_ms_256": 0.3234824999992725 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.7800044908498933, "recall1": 0.7355731447176378, "recall3": 0.928146401706523, "recall5": 0.9529864151790727, "recall8": 0.9671606601549343, "evidence_count_in_top3": 1.0318288986190636, "hit3": 0.936510609632873, "mrr": 0.8612520819605932, "ndcg3": 0.8691557053195522, "mean_first_positive_rank": 1.9977545750533288, "all_evidence_in_top3": 0.9197821937801729, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.849, "hop_accuracy": 0.8175182481751825, "hop_accuracy_answerable": 0.8877287526664421, "hop_under_prediction": 0.11227124733355788, "hop_over_prediction": 0.0, "mean_score_margin": 53.160489893678886, "median_score_margin": 48.923973083496094, "negative_margin_rate": 0.21999550915010666, "hop_confusion": { "true_0_pred_0": 2106, "true_0_pred_1": 2000, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 2000, "true_2_pred_1": 0, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.8877236850807078, "unknown_mean_need_prob": 0.4528151513680766, "thr0.30": { "need_precision": 0.8877287526664421, "need_recall": 0.8877287526664421, "need_f1": 0.8877287526664421, "specificity_unknown_refusal": 0.5129079396005846, "abstention_accuracy": 0.8175182481751825, "known_question_refusal_rate": 0.11227124733355788, "unknown_question_read_rate": 0.4870920603994155, "counts": { "tp": 15814, "tn": 2106, "fp": 2000, "fn": 2000 } }, "thr0.40": { "need_precision": 0.8877287526664421, "need_recall": 0.8877287526664421, "need_f1": 0.8877287526664421, "specificity_unknown_refusal": 0.5129079396005846, "abstention_accuracy": 0.8175182481751825, "known_question_refusal_rate": 0.11227124733355788, "unknown_question_read_rate": 0.4870920603994155, "counts": { "tp": 15814, "tn": 2106, "fp": 2000, "fn": 2000 } }, "thr0.50": { "need_precision": 0.8877287526664421, "need_recall": 0.8877287526664421, "need_f1": 0.8877287526664421, "specificity_unknown_refusal": 0.5129079396005846, "abstention_accuracy": 0.8175182481751825, "known_question_refusal_rate": 0.11227124733355788, "unknown_question_read_rate": 0.4870920603994155, "counts": { "tp": 15814, "tn": 2106, "fp": 2000, "fn": 2000 } }, "thr0.60": { "need_precision": 0.9022136010953903, "need_recall": 0.8877287526664421, "need_f1": 0.8949125686152454, "specificity_unknown_refusal": 0.5825621042377009, "abstention_accuracy": 0.8305656934306569, "known_question_refusal_rate": 0.11227124733355788, "unknown_question_read_rate": 0.41743789576229906, "counts": { "tp": 15814, "tn": 2392, "fp": 1714, "fn": 2000 } }, "thr0.70": { "need_precision": 0.9022136010953903, "need_recall": 0.8877287526664421, "need_f1": 0.8949125686152454, "specificity_unknown_refusal": 0.5825621042377009, "abstention_accuracy": 0.8305656934306569, "known_question_refusal_rate": 0.11227124733355788, "unknown_question_read_rate": 0.41743789576229906, "counts": { "tp": 15814, "tn": 2392, "fp": 1714, "fn": 2000 } }, "thr0.80": { "need_precision": 0.9022136010953903, "need_recall": 0.8877287526664421, "need_f1": 0.8949125686152454, "specificity_unknown_refusal": 0.5825621042377009, "abstention_accuracy": 0.8305656934306569, "known_question_refusal_rate": 0.11227124733355788, "unknown_question_read_rate": 0.41743789576229906, "counts": { "tp": 15814, "tn": 2392, "fp": 1714, "fn": 2000 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 8.474685865469522e-08, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.8178125, "recall1": 0.76834375, "recall3": 0.9543125, "recall5": 0.96865625, "recall8": 0.9719375, "evidence_count_in_top3": 1.06975, "hit3": 0.963625, "mrr": 0.8901514232628588, "ndcg3": 0.9005039309554683, "mean_first_positive_rank": 1.859375, "all_evidence_in_top3": 0.945, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.849, "hop_accuracy": 0.8, "hop_accuracy_answerable": 0.875, "hop_under_prediction": 0.125, "hop_over_prediction": 0.0, "mean_score_margin": 59.239713783336335, "median_score_margin": 56.72071409225464, "negative_margin_rate": 0.1821875, "hop_confusion": { "true_0_pred_0": 2000, "true_0_pred_1": 2000, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 2000, "true_2_pred_1": 0, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.8749969797285945, "unknown_mean_need_prob": 0.4644220104073552, "thr0.50": { "need_precision": 0.875, "need_recall": 0.875, "need_f1": 0.875, "specificity_unknown_refusal": 0.5, "abstention_accuracy": 0.8, "known_question_refusal_rate": 0.125, "unknown_question_read_rate": 0.5, "counts": { "tp": 14000, "tn": 2000, "fp": 2000, "fn": 2000 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.44639718804920914, "recall1": 0.44639718804920914, "recall3": 0.6942003514938488, "recall5": 0.8096074985354423, "recall8": 0.9209138840070299, "evidence_count_in_top3": 0.6942003514938488, "hit3": 0.6942003514938488, "mrr": 0.6050143063165109, "ndcg3": 0.5907781161122315, "mean_first_positive_rank": 3.256590509666081, "all_evidence_in_top3": 0.6942003514938488, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.016779011952367244, "median_score_margin": -1.933023452758789, "negative_margin_rate": 0.5536028119507909, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999754249386154, "unknown_mean_need_prob": 0.03740391581197314, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.4485981308411215, "recall1": 0.4485981308411215, "recall3": 0.7476635514018691, "recall5": 0.897196261682243, "recall8": 0.9906542056074766, "evidence_count_in_top3": 0.7476635514018691, "hit3": 0.7476635514018691, "mrr": 0.6276859341867589, "ndcg3": 0.6226036922563473, "mean_first_positive_rank": 2.607476635514019, "all_evidence_in_top3": 0.7476635514018691, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -7.530951345635351, "median_score_margin": -5.711280822753906, "negative_margin_rate": 0.5514018691588785, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.03740391581197314, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.8025, "recall1": 0.8025, "recall3": 0.998, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 0.998, "hit3": 0.998, "mrr": 0.8965833333333334, "ndcg3": 0.9225735229839335, "mean_first_positive_rank": 1.2265, "all_evidence_in_top3": 0.998, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 60.32892245580256, "median_score_margin": 50.02846574783325, "negative_margin_rate": 0.1975, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999776878654957, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.898, "recall1": 0.898, "recall3": 0.9995, "recall5": 0.9995, "recall8": 0.9995, "evidence_count_in_top3": 0.9995, "hit3": 0.9995, "mrr": 0.9473787878787878, "ndcg3": 0.9609264670821456, "mean_first_positive_rank": 1.115, "all_evidence_in_top3": 0.9995, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 63.02626673298329, "median_score_margin": 64.21059799194336, "negative_margin_rate": 0.102, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999966541528702, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.906, "recall1": 0.906, "recall3": 0.9995, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 0.9995, "hit3": 0.9995, "mrr": 0.9512916666666666, "ndcg3": 0.9637480993000024, "mean_first_positive_rank": 1.1045, "all_evidence_in_top3": 0.9995, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 63.26017077337205, "median_score_margin": 64.34984970092773, "negative_margin_rate": 0.094, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999967547059059, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.9287911344170571, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 1.0, "counts": { "tp": 0, "tn": 0, "fp": 2000, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.6965, "recall1": 0.6965, "recall3": 0.8465, "recall5": 0.8605, "recall8": 0.8695, "evidence_count_in_top3": 0.8465, "hit3": 0.8465, "mrr": 0.776365632860676, "ndcg3": 0.7873425001821464, "mean_first_positive_rank": 4.702, "all_evidence_in_top3": 0.8465, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 59.559096137337384, "median_score_margin": 77.47207641601562, "negative_margin_rate": 0.3035, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.7915, "recall1": 0.39575, "recall3": 0.9235, "recall5": 0.99575, "recall8": 1.0, "evidence_count_in_top3": 1.847, "hit3": 0.998, "mrr": 0.891, "ndcg3": 0.8664546282365805, "mean_first_positive_rank": 1.238, "all_evidence_in_top3": 0.849, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.849, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": 34.953358390808106, "median_score_margin": 41.703819274902344, "negative_margin_rate": 0.2085, "hop_confusion": { "true_2_pred_0": 2000, "true_2_pred_2": 0 }, "known_mean_need_prob": 9.34347713071304e-06, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": null, "need_recall": 0.0, "need_f1": null, "specificity_unknown_refusal": null, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": null, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2000 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.625, "recall1": 0.625, "recall3": 0.8685, "recall5": 0.8935, "recall8": 0.9065, "evidence_count_in_top3": 0.8685, "hit3": 0.8685, "mrr": 0.7501752986967395, "ndcg3": 0.7711683990410768, "mean_first_positive_rank": 3.293, "all_evidence_in_top3": 0.8685, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 22.522678687380626, "median_score_margin": 15.791908264160156, "negative_margin_rate": 0.375, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.999996542096138, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.92, "recall1": 0.92, "recall3": 0.999, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 0.999, "hit3": 0.999, "mrr": 0.9584166666666666, "ndcg3": 0.9687960125035735, "mean_first_positive_rank": 1.09, "all_evidence_in_top3": 0.999, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 101.63241866916046, "median_score_margin": 87.8995840549469, "negative_margin_rate": 0.08, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999988555312157, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.903, "recall1": 0.903, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.95, "ndcg3": 0.9630218183142882, "mean_first_positive_rank": 1.106, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 68.63479841984622, "median_score_margin": 62.0030552148819, "negative_margin_rate": 0.097, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.44652701212789414, "recall1": 0.44652701212789414, "recall3": 0.697353914002205, "recall5": 0.814773980154355, "recall8": 0.9250275633958104, "evidence_count_in_top3": 0.697353914002205, "hit3": 0.697353914002205, "mrr": 0.606351607409188, "ndcg3": 0.5926553689498392, "mean_first_positive_rank": 3.2183020948180814, "all_evidence_in_top3": 0.697353914002205, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.46000747926442864, "median_score_margin": -2.0159082412719727, "negative_margin_rate": 0.5534729878721059, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999768745150035, "unknown_mean_need_prob": 0.01482047064058326, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 5.2886397653310265e-05, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "V2-512 v6 best": { "checkpoint": "checkpoints\\router_v6_v2_512\\router_best.pt", "router": { "kind": "router_v2", "step": 2000, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3 }, "parameters": 4741902, "checkpoint_bytes": 56922573 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048, "checkpoint_bytes": 56922573 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.8750353333986519, "single_query_latency_ms_p50": 0.8467999978165608, "single_query_latency_ms_p95": 1.2185999985376839, "queries_per_second": 1142.8109949754637, "samples": 150, "batched_qps_64": 73617.37369428946, "batched_latency_ms_64": 0.013583750001089356, "batched_qps_256": 161941.27100652116, "batched_latency_ms_256": 0.006175078124215361 }, "cpu": { "single_query_latency_ms_mean": 0.921385333276703, "single_query_latency_ms_p50": 0.8951000017987099, "single_query_latency_ms_p95": 1.1334999981045257, "queries_per_second": 1085.3222467126986, "samples": 150, "batched_qps_64": 2924.674104534912, "batched_latency_ms_64": 0.3419184374934048, "batched_qps_256": 3073.5827721960004, "batched_latency_ms_256": 0.32535320312376825 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.950488379925901, "recall1": 0.8971034018187942, "recall3": 0.9884079937128102, "recall5": 0.9934882676546536, "recall8": 0.9961266419669923, "evidence_count_in_top3": 1.0986302907825305, "hit3": 0.9904569439766476, "mrr": 0.9711571604054701, "ndcg3": 0.9723689530227545, "mean_first_positive_rank": 1.1156393847535646, "all_evidence_in_top3": 0.9863590434489727, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9635, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 12.36926158705623, "median_score_margin": 12.002254128456116, "negative_margin_rate": 0.049511620074099025, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9988408471058552, "unknown_mean_need_prob": 0.006478862109410833, "thr0.30": { "need_precision": 0.999326825984517, "need_recall": 1.0, "need_f1": 0.9996632996632996, "specificity_unknown_refusal": 0.9970774476376035, "abstention_accuracy": 0.9994525547445255, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0029225523623964927, "counts": { "tp": 17814, "tn": 4094, "fp": 12, "fn": 0 } }, "thr0.40": { "need_precision": 0.999326825984517, "need_recall": 1.0, "need_f1": 0.9996632996632996, "specificity_unknown_refusal": 0.9970774476376035, "abstention_accuracy": 0.9994525547445255, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0029225523623964927, "counts": { "tp": 17814, "tn": 4094, "fp": 12, "fn": 0 } }, "thr0.50": { "need_precision": 0.9995511165974638, "need_recall": 1.0, "need_f1": 0.999775507913346, "specificity_unknown_refusal": 0.9980516317584024, "abstention_accuracy": 0.9996350364963503, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.001948368241597662, "counts": { "tp": 17814, "tn": 4098, "fp": 8, "fn": 0 } }, "thr0.60": { "need_precision": 0.9998877413560844, "need_recall": 1.0, "need_f1": 0.9999438675273646, "specificity_unknown_refusal": 0.9995129079396006, "abstention_accuracy": 0.9999087591240876, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0004870920603994155, "counts": { "tp": 17814, "tn": 4104, "fp": 2, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.0904978112084791, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.97, "recall1": 0.9105625, "recall3": 0.99771875, "recall5": 0.9999375, "recall8": 1.0, "evidence_count_in_top3": 1.1204375, "hit3": 1.0, "mrr": 0.9849166666666667, "ndcg3": 0.9857856193167982, "mean_first_positive_rank": 1.0305, "all_evidence_in_top3": 0.9954375, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9635, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 13.542529965203517, "median_score_margin": 12.727105312049389, "negative_margin_rate": 0.03, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9990760059095919, "unknown_mean_need_prob": 0.0025661538663334796, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.8125366139425894, "recall1": 0.8125366139425894, "recall3": 0.9244288224956063, "recall5": 0.9502050380785003, "recall8": 0.971294669009959, "evidence_count_in_top3": 0.9244288224956063, "hit3": 0.9244288224956063, "mrr": 0.8742941912651278, "ndcg3": 0.878837443896981, "mean_first_positive_rank": 1.7176332747510252, "all_evidence_in_top3": 0.9244288224956063, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.2332465364638656, "median_score_margin": 2.495278835296631, "negative_margin_rate": 0.18746338605741067, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.996912036259131, "unknown_mean_need_prob": 0.2510888675848643, "thr0.50": { "need_precision": 0.9976621858562245, "need_recall": 1.0, "need_f1": 0.9988297249853716, "specificity_unknown_refusal": 0.8095238095238095, "abstention_accuracy": 0.9976851851851852, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.19047619047619047, "counts": { "tp": 1707, "tn": 17, "fp": 4, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.2336448598130841, "recall1": 0.2336448598130841, "recall3": 0.616822429906542, "recall5": 0.719626168224299, "recall8": 0.8130841121495327, "evidence_count_in_top3": 0.616822429906542, "hit3": 0.616822429906542, "mrr": 0.4589420963252739, "ndcg3": 0.4582719938918631, "mean_first_positive_rank": 4.242990654205608, "all_evidence_in_top3": 0.616822429906542, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -1.3701436372003823, "median_score_margin": -0.6190404891967773, "negative_margin_rate": 0.7663551401869159, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 0.9944477560364198, "unknown_mean_need_prob": 0.2510888675848643, "thr0.50": { "need_precision": 0.963963963963964, "need_recall": 1.0, "need_f1": 0.9816513761467891, "specificity_unknown_refusal": 0.8095238095238095, "abstention_accuracy": 0.96875, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.19047619047619047, "counts": { "tp": 107, "tn": 17, "fp": 4, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9905, "recall1": 0.9905, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99525, "ndcg3": 0.9964938326589288, "mean_first_positive_rank": 1.0095, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 16.784344019554556, "median_score_margin": 14.332054734230042, "negative_margin_rate": 0.0095, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9984212898612023, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.974, "recall1": 0.974, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.987, "ndcg3": 0.9904041735928579, "mean_first_positive_rank": 1.026, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 14.32004562981613, "median_score_margin": 12.83707270026207, "negative_margin_rate": 0.026, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9998619016408921, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.969, "recall1": 0.969, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9844166666666666, "ndcg3": 0.9884933574839294, "mean_first_positive_rank": 1.0315, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 14.515158921549562, "median_score_margin": 12.90774929523468, "negative_margin_rate": 0.031, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9998633873462677, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.004985218498797622, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.973, "recall1": 0.973, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9864166666666666, "ndcg3": 0.9899696384696437, "mean_first_positive_rank": 1.0275, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 11.82059130709432, "median_score_margin": 12.446841090917587, "negative_margin_rate": 0.027, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9998804149031639, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.951, "recall1": 0.4755, "recall3": 0.98175, "recall5": 0.9995, "recall8": 1.0, "evidence_count_in_top3": 1.9635, "hit3": 1.0, "mrr": 0.9753333333333334, "ndcg3": 0.9571701608629515, "mean_first_positive_rank": 1.05, "all_evidence_in_top3": 0.9635, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9635, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 10.0001081677787, "median_score_margin": 11.238629460334778, "negative_margin_rate": 0.049, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9978087923228741, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9755, "recall1": 0.9755, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98775, "ndcg3": 0.9909577789625007, "mean_first_positive_rank": 1.0245, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 14.530858453191817, "median_score_margin": 12.788989305496216, "negative_margin_rate": 0.0245, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9972889910042286, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9745, "recall1": 0.9745, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9871666666666666, "ndcg3": 0.9905232438392865, "mean_first_positive_rank": 1.026, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 13.56638970718137, "median_score_margin": 13.154640913009644, "negative_margin_rate": 0.0255, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9995595035254955, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9525, "recall1": 0.9525, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.976, "ndcg3": 0.982272768664287, "mean_first_positive_rank": 1.049, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 12.802743515461684, "median_score_margin": 12.365995168685913, "negative_margin_rate": 0.0475, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999237666726113, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.7783902976846747, "recall1": 0.7783902976846747, "recall3": 0.906284454244763, "recall5": 0.9366041896361632, "recall8": 0.961962513781698, "evidence_count_in_top3": 0.906284454244763, "hit3": 0.906284454244763, "mrr": 0.8497943708910569, "ndcg3": 0.8540301102968997, "mean_first_positive_rank": 1.866593164277839, "all_evidence_in_top3": 0.906284454244763, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.0206981634858754, "median_score_margin": 2.3058557510375977, "negative_margin_rate": 0.22160970231532526, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9967666790464352, "unknown_mean_need_prob": 0.1541282297727072, "thr0.50": { "need_precision": 0.995609220636663, "need_recall": 1.0, "need_f1": 0.9977997799779977, "specificity_unknown_refusal": 0.9245283018867925, "abstention_accuracy": 0.9958333333333333, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.07547169811320754, "counts": { "tp": 1814, "tn": 98, "fp": 8, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.00014708923386933748, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "V2-512 v6 final": { "checkpoint": "checkpoints\\router_v6_v2_512\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3 }, "parameters": 4741902, "checkpoint_bytes": 18973433 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048, "checkpoint_bytes": 18973433 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.8725079999809774, "single_query_latency_ms_p50": 0.850600001285784, "single_query_latency_ms_p95": 1.0982999992847908, "queries_per_second": 1146.121296334019, "samples": 150, "batched_qps_64": 61636.8434007473, "batched_latency_ms_64": 0.016224062505898473, "batched_qps_256": 164167.81025689258, "batched_latency_ms_256": 0.006091328125990003 }, "cpu": { "single_query_latency_ms_mean": 0.8932059997459874, "single_query_latency_ms_p50": 0.876300000527408, "single_query_latency_ms_p95": 1.1406000012357254, "queries_per_second": 1119.5625648331775, "samples": 150, "batched_qps_64": 2943.6168587607826, "batched_latency_ms_64": 0.3397181250079484, "batched_qps_256": 3143.215455374193, "batched_latency_ms_256": 0.3181455468762806 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.941843493881217, "recall1": 0.8864657011339396, "recall3": 0.9666273717300999, "recall5": 0.9764230380599529, "recall8": 0.984506567867969, "evidence_count_in_top3": 1.07668126192882, "hit3": 0.9688447288649377, "mrr": 0.958724954839147, "ndcg3": 0.9548399733910118, "mean_first_positive_rank": 1.2825867295385651, "all_evidence_in_top3": 0.9644100145952621, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9605, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 73.24820388849786, "median_score_margin": 76.59281730651855, "negative_margin_rate": 0.05815650611878298, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999984274064, "unknown_mean_need_prob": 4.522551017238816e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 4.432710963581804e-06, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.98375, "recall1": 0.92209375, "recall3": 0.99753125, "recall5": 0.9999375, "recall8": 1.0, "evidence_count_in_top3": 1.1200625, "hit3": 1.0, "mrr": 0.99184375, "ndcg3": 0.9902417180168516, "mean_first_positive_rank": 1.0164375, "all_evidence_in_top3": 0.9950625, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9605, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 81.85253369444608, "median_score_margin": 83.23931503295898, "negative_margin_rate": 0.01625, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999701977, "unknown_mean_need_prob": 6.058075942262249e-11, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.5770357352079672, "recall1": 0.5770357352079672, "recall3": 0.6859988283538372, "recall5": 0.7562975981253661, "recall8": 0.8388986526069128, "evidence_count_in_top3": 0.6859988283538372, "hit3": 0.6859988283538372, "mrr": 0.6663638286240626, "ndcg3": 0.6401845706816489, "mean_first_positive_rank": 3.719976567076743, "all_evidence_in_top3": 0.6859988283538372, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -2.648404171182946, "median_score_margin": 5.544002532958984, "negative_margin_rate": 0.4229642647920328, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999838679872, "unknown_mean_need_prob": 0.00043537282504942034, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.4953271028037383, "recall1": 0.4953271028037383, "recall3": 0.822429906542056, "recall5": 0.9719626168224299, "recall8": 0.9906542056074766, "evidence_count_in_top3": 0.822429906542056, "hit3": 0.822429906542056, "mrr": 0.6704980377877574, "ndcg3": 0.680904070694264, "mean_first_positive_rank": 2.196261682242991, "all_evidence_in_top3": 0.822429906542056, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -2.581393656329574, "median_score_margin": -0.056890010833740234, "negative_margin_rate": 0.5046728971962616, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.00043537282504942034, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9935, "recall1": 0.9935, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99675, "ndcg3": 0.9976010433982145, "mean_first_positive_rank": 1.0065, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 64.64610423742235, "median_score_margin": 60.52951920032501, "negative_margin_rate": 0.0065, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.98, "recall1": 0.98, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99, "ndcg3": 0.9926185950714291, "mean_first_positive_rank": 1.02, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 88.94486605204642, "median_score_margin": 95.25923618674278, "negative_margin_rate": 0.02, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9805, "recall1": 0.9805, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9901666666666666, "ndcg3": 0.9927376653178577, "mean_first_positive_rank": 1.02, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 90.02319332621246, "median_score_margin": 96.11362898349762, "negative_margin_rate": 0.0195, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.2115745555928435e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.977, "recall1": 0.977, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9884166666666666, "ndcg3": 0.9914459194553578, "mean_first_positive_rank": 1.0235, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 78.39938361007721, "median_score_margin": 81.58803009986877, "negative_margin_rate": 0.023, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9865, "recall1": 0.49325, "recall3": 0.98025, "recall5": 0.9995, "recall8": 1.0, "evidence_count_in_top3": 1.9605, "hit3": 1.0, "mrr": 0.99325, "ndcg3": 0.9651268224740949, "mean_first_positive_rank": 1.0135, "all_evidence_in_top3": 0.9605, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9605, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 104.36254599650204, "median_score_margin": 118.10550880432129, "negative_margin_rate": 0.0135, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999997615814, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9955, "recall1": 0.9955, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99775, "ndcg3": 0.9983391838910716, "mean_first_positive_rank": 1.0045, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 57.61693591082841, "median_score_margin": 57.403422355651855, "negative_margin_rate": 0.0045, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9825, "recall1": 0.9825, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99125, "ndcg3": 0.9935412706875005, "mean_first_positive_rank": 1.0175, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 84.6615379076004, "median_score_margin": 90.02162456512451, "negative_margin_rate": 0.0175, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9745, "recall1": 0.9745, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9871666666666666, "ndcg3": 0.9905232438392865, "mean_first_positive_rank": 1.026, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 86.1657025148794, "median_score_margin": 91.7922420501709, "negative_margin_rate": 0.0255, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.5722160970231532, "recall1": 0.5722160970231532, "recall3": 0.6940463065049615, "recall5": 0.7690187431091511, "recall8": 0.8478500551267916, "evidence_count_in_top3": 0.6940463065049615, "hit3": 0.6940463065049615, "mrr": 0.666607687709242, "ndcg3": 0.6425864375511913, "mean_first_positive_rank": 3.630099228224917, "all_evidence_in_top3": 0.6940463065049615, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -2.644451511266016, "median_score_margin": 4.475192725658417, "negative_margin_rate": 0.42778390297684676, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999848195447, "unknown_mean_need_prob": 0.00017518256748815932, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 4.063285960620704e-15, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "XL-512 v6 best": { "checkpoint": "checkpoints\\router_v6_xl512\\router_best.pt", "router": { "kind": "router_xl", "step": 36000, "arch_config": { "arch": "router_xl", "arch_version": 1, "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3, "encoder_layers": 2, "encoder_hidden": 512, "pair_blocks": 1, "pair_hidden": 512, "pair_expansion": 2, "pair_dropout": 0.05, "use_interaction": true, "policy_layers": 2, "policy_hidden": 512, "policy_dropout": 0.0, "learnable_cosine_scale": true }, "parameters": 7898127, "checkpoint_bytes": 99014505 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048, "checkpoint_bytes": 99014505 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.2161159999474573, "single_query_latency_ms_p50": 1.1961999989580363, "single_query_latency_ms_p95": 1.5090000015334226, "queries_per_second": 822.2899789520122, "samples": 150, "batched_qps_64": 57386.75086015149, "batched_latency_ms_64": 0.017425624992029043, "batched_qps_256": 96746.88596045674, "batched_latency_ms_256": 0.010336249999909342 }, "cpu": { "single_query_latency_ms_mean": 1.6496533333702246, "single_query_latency_ms_p50": 1.6746999972383492, "single_query_latency_ms_p95": 2.007999999477761, "queries_per_second": 606.1879667511782, "samples": 150, "batched_qps_64": 1660.2789164532574, "batched_latency_ms_64": 0.6023084375101462, "batched_qps_256": 1699.1487264831305, "batched_latency_ms_256": 0.5885300000016969 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.957842146626249, "recall1": 0.9024082182553048, "recall3": 0.9758897496351184, "recall5": 0.9834399910183003, "recall8": 0.9905411474121477, "evidence_count_in_top3": 1.0808914337038285, "hit3": 0.9831593128999663, "mrr": 0.9724377931581535, "ndcg3": 0.9664212737428025, "mean_first_positive_rank": 1.1337150555742674, "all_evidence_in_top3": 0.9686201863702706, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.8705, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 45.56406442258217, "median_score_margin": 47.89992094039917, "negative_margin_rate": 0.042157853373750984, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999977515257, "unknown_mean_need_prob": 9.165001448203187e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.130629900134017e-05, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.98375, "recall1": 0.92203125, "recall3": 0.99190625, "recall5": 0.994875, "recall8": 0.99640625, "evidence_count_in_top3": 1.1088125, "hit3": 1.0, "mrr": 0.99184375, "ndcg3": 0.9863265131728122, "mean_first_positive_rank": 1.0164375, "all_evidence_in_top3": 0.9838125, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.8705, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 48.28189839535998, "median_score_margin": 49.04537320137024, "negative_margin_rate": 0.01625, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999552965, "unknown_mean_need_prob": 2.862315262235959e-10, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.7457527826596367, "recall1": 0.7457527826596367, "recall3": 0.8347978910369068, "recall5": 0.8763913298183948, "recall8": 0.9355594610427651, "evidence_count_in_top3": 0.8347978910369068, "hit3": 0.8347978910369068, "mrr": 0.8100220298155272, "ndcg3": 0.7979455040170743, "mean_first_positive_rank": 2.16813122437024, "all_evidence_in_top3": 0.8347978910369068, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 22.8850553075215, "median_score_margin": 27.899999618530273, "negative_margin_rate": 0.2542472173403632, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999769542673, "unknown_mean_need_prob": 0.0008787320924793242, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.4672897196261682, "recall1": 0.4672897196261682, "recall3": 0.8317757009345794, "recall5": 0.9813084112149533, "recall8": 0.9906542056074766, "evidence_count_in_top3": 0.8317757009345794, "hit3": 0.8317757009345794, "mrr": 0.6616751628433871, "ndcg3": 0.6776764890854534, "mean_first_positive_rank": 2.1682242990654204, "all_evidence_in_top3": 0.8317757009345794, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.9633634409336286, "median_score_margin": -1.062392234802246, "negative_margin_rate": 0.5327102803738317, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0008787320924793242, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.992, "recall1": 0.992, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.996, "ndcg3": 0.9970474380285717, "mean_first_positive_rank": 1.008, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 67.69295654568822, "median_score_margin": 69.3216142654419, "negative_margin_rate": 0.008, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9795, "recall1": 0.9795, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98975, "ndcg3": 0.9924340599482149, "mean_first_positive_rank": 1.0205, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 39.98083441020921, "median_score_margin": 44.252100229263306, "negative_margin_rate": 0.0205, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9805, "recall1": 0.9805, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99025, "ndcg3": 0.9928031301946435, "mean_first_positive_rank": 1.0195, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 40.70788796337694, "median_score_margin": 44.594672441482544, "negative_margin_rate": 0.0195, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 5.724454182644648e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9815, "recall1": 0.9815, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9906666666666666, "ndcg3": 0.9931067355642863, "mean_first_positive_rank": 1.019, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 37.93500039816182, "median_score_margin": 40.258564949035645, "negative_margin_rate": 0.0185, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9875, "recall1": 0.49375, "recall3": 0.93525, "recall5": 0.959, "recall8": 0.97125, "evidence_count_in_top3": 1.8705, "hit3": 1.0, "mrr": 0.9936666666666666, "ndcg3": 0.9341087890914226, "mean_first_positive_rank": 1.013, "all_evidence_in_top3": 0.8705, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.8705, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 47.13378771021217, "median_score_margin": 51.072168588638306, "negative_margin_rate": 0.0125, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999996423722, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9965, "recall1": 0.9965, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99825, "ndcg3": 0.9987082541375002, "mean_first_positive_rank": 1.0035, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 61.5167154010348, "median_score_margin": 62.94392251968384, "negative_margin_rate": 0.0035, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9815, "recall1": 0.9815, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99075, "ndcg3": 0.9931722004410719, "mean_first_positive_rank": 1.0185, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 38.85421975574363, "median_score_margin": 43.87380111217499, "negative_margin_rate": 0.0185, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.971, "recall1": 0.971, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9854166666666666, "ndcg3": 0.9892314979767864, "mean_first_positive_rank": 1.0295, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 52.43378497845307, "median_score_margin": 59.86355948448181, "negative_margin_rate": 0.029, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.7293274531422271, "recall1": 0.7293274531422271, "recall3": 0.834619625137817, "recall5": 0.88257993384785, "recall8": 0.9388092613009923, "evidence_count_in_top3": 0.834619625137817, "hit3": 0.834619625137817, "mrr": 0.8012716909147449, "ndcg3": 0.7908513559477891, "mean_first_positive_rank": 2.1681367144432193, "all_evidence_in_top3": 0.834619625137817, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 21.591989690253087, "median_score_margin": 24.97784423828125, "negative_margin_rate": 0.27067254685777287, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999783136353, "unknown_mean_need_prob": 0.0003550033115114848, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.7634182726834057e-14, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "XL-512 v6 final": { "checkpoint": "checkpoints\\router_v6_xl512\\memory_router_xl.pt", "router": { "kind": "router_xl", "step": null, "arch_config": { "arch": "router_xl", "arch_version": 1, "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3, "encoder_layers": 2, "encoder_hidden": 512, "pair_blocks": 1, "pair_hidden": 512, "pair_expansion": 2, "pair_dropout": 0.05, "use_interaction": true, "policy_layers": 2, "policy_hidden": 512, "policy_dropout": 0.0, "learnable_cosine_scale": true }, "parameters": 7898127, "checkpoint_bytes": 35802155 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048, "checkpoint_bytes": 35802155 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.2505186665900208, "single_query_latency_ms_p50": 1.2239999996381812, "single_query_latency_ms_p95": 1.6236000010394491, "queries_per_second": 799.6681910609555, "samples": 150, "batched_qps_64": 53291.58828255585, "batched_latency_ms_64": 0.018764687490602228, "batched_qps_256": 94345.88082639035, "batched_latency_ms_256": 0.010599296876989683 }, "cpu": { "single_query_latency_ms_mean": 1.717839333335481, "single_query_latency_ms_p50": 1.7062999977497384, "single_query_latency_ms_p95": 2.0226999986334704, "queries_per_second": 582.1266171955253, "samples": 150, "batched_qps_64": 1690.8261059433491, "batched_latency_ms_64": 0.5914268749961593, "batched_qps_256": 1707.8817676137278, "batched_latency_ms_256": 0.585520625000413 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9502077018075671, "recall1": 0.8948299090602897, "recall3": 0.965336252385764, "recall5": 0.9760862243179522, "recall8": 0.9854608734703043, "evidence_count_in_top3": 1.0657348153137982, "hit3": 0.9772089367912877, "mrr": 0.9664213223997663, "ndcg3": 0.9560053718886551, "mean_first_positive_rank": 1.1716066015493432, "all_evidence_in_top3": 0.9534635679802402, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.7885, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 215.75331382084113, "median_score_margin": 239.68688774108887, "negative_margin_rate": 0.049792298192432916, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999979790499, "unknown_mean_need_prob": 4.1244088529706584e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 4.264552800687521e-06, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.984, "recall1": 0.92234375, "recall3": 0.98678125, "recall5": 0.990875, "recall8": 0.9933125, "evidence_count_in_top3": 1.0985625, "hit3": 1.0, "mrr": 0.99196875, "ndcg3": 0.9823026449115756, "mean_first_positive_rank": 1.0161875, "all_evidence_in_top3": 0.9735625, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.7885, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 235.8910635419197, "median_score_margin": 250.7837634086609, "negative_margin_rate": 0.016, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999627471, "unknown_mean_need_prob": 7.011441629130955e-11, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.6690099589923842, "recall1": 0.6690099589923842, "recall3": 0.7709431751611013, "recall5": 0.8388986526069128, "recall8": 0.9121265377855887, "evidence_count_in_top3": 0.7709431751611013, "hit3": 0.7709431751611013, "mrr": 0.7488416468644941, "ndcg3": 0.7281836459473614, "mean_first_positive_rank": 2.558289396602226, "all_evidence_in_top3": 0.7709431751611013, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 40.85260479970152, "median_score_margin": 62.14678978919983, "negative_margin_rate": 0.3309900410076157, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999792588407, "unknown_mean_need_prob": 0.00039670502174734185, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.38317757009345793, "recall1": 0.38317757009345793, "recall3": 0.8598130841121495, "recall5": 0.9532710280373832, "recall8": 0.9813084112149533, "evidence_count_in_top3": 0.8598130841121495, "hit3": 0.8598130841121495, "mrr": 0.617352766651832, "ndcg3": 0.6582046038050816, "mean_first_positive_rank": 2.289719626168224, "all_evidence_in_top3": 0.8598130841121495, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -5.260566909736562, "median_score_margin": -7.525552272796631, "negative_margin_rate": 0.616822429906542, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.00039670502174734185, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9925, "recall1": 0.9925, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99625, "ndcg3": 0.9972319731517859, "mean_first_positive_rank": 1.0075, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 272.5490972074904, "median_score_margin": 288.63626432418823, "negative_margin_rate": 0.0075, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.98, "recall1": 0.98, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99, "ndcg3": 0.9926185950714291, "mean_first_positive_rank": 1.02, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 215.3791910160221, "median_score_margin": 239.17540168762207, "negative_margin_rate": 0.02, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.982, "recall1": 0.982, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9909166666666667, "ndcg3": 0.9932912706875006, "mean_first_positive_rank": 1.0185, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 219.1182684260085, "median_score_margin": 239.48388767242432, "negative_margin_rate": 0.018, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.4022628460683615e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.983, "recall1": 0.983, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9914166666666666, "ndcg3": 0.9936603409339291, "mean_first_positive_rank": 1.0175, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 211.44927242165804, "median_score_margin": 233.2896122932434, "negative_margin_rate": 0.017, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9865, "recall1": 0.49325, "recall3": 0.89425, "recall5": 0.927, "recall8": 0.9465, "evidence_count_in_top3": 1.7885, "hit3": 1.0, "mrr": 0.99325, "ndcg3": 0.9008760971390303, "mean_first_positive_rank": 1.0135, "all_evidence_in_top3": 0.7885, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.7885, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 274.43398497897016, "median_score_margin": 301.56740951538086, "negative_margin_rate": 0.0135, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999997019767, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.995, "recall1": 0.995, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9975, "ndcg3": 0.9981546487678573, "mean_first_positive_rank": 1.005, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 223.85322645658067, "median_score_margin": 222.2765736579895, "negative_margin_rate": 0.005, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9805, "recall1": 0.9805, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99025, "ndcg3": 0.9928031301946435, "mean_first_positive_rank": 1.0195, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 225.64811652239038, "median_score_margin": 241.96742022037506, "negative_margin_rate": 0.0195, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9725, "recall1": 0.9725, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9861666666666666, "ndcg3": 0.9897851033464293, "mean_first_positive_rank": 1.028, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 244.6973513062373, "median_score_margin": 263.6290616989136, "negative_margin_rate": 0.0275, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.6521499448732084, "recall1": 0.6521499448732084, "recall3": 0.7761852260198456, "recall5": 0.8456449834619625, "recall8": 0.9162072767364939, "evidence_count_in_top3": 0.7761852260198456, "hit3": 0.7761852260198456, "mrr": 0.7410856875575729, "ndcg3": 0.7240558854681862, "mean_first_positive_rank": 2.54244762954796, "all_evidence_in_top3": 0.7761852260198456, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 38.1325886073587, "median_score_margin": 54.42159175872803, "negative_margin_rate": 0.3478500551267916, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999804822718, "unknown_mean_need_prob": 0.0001597598329493619, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 2.5479757829358922e-15, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } } }