{ "V2-128 deployed(v3)": { "checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_qwen_router_entities\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 8157561 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 8157561 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.0995226665302955, "single_query_latency_ms_p50": 0.9548999951221049, "single_query_latency_ms_p95": 2.3265999989234842, "queries_per_second": 909.4855708210602, "samples": 150, "batched_qps_64": 66037.93061957207, "batched_latency_ms_64": 0.0151428124809172, "batched_qps_256": 207287.4493168988, "batched_latency_ms_256": 0.00482421875176442 }, "cpu": { "single_query_latency_ms_mean": 0.4118146665374904, "single_query_latency_ms_p50": 0.3815000018221326, "single_query_latency_ms_p95": 0.6266999989748001, "queries_per_second": 2428.276798415005, "samples": 150, "batched_qps_64": 11685.442492836237, "batched_latency_ms_64": 0.0855765625146887, "batched_qps_256": 12349.573987243031, "batched_latency_ms_256": 0.08097445312955642 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.4112495789828225, "recall1": 0.37027057370607386, "recall3": 0.4673009992141013, "recall5": 0.4891377568204783, "recall8": 0.5160267205568654, "evidence_count_in_top3": 0.5420455821264174, "hit3": 0.5003368137420007, "mrr": 0.476908110965675, "ndcg3": 0.44220238482445295, "mean_first_positive_rank": 14.38671831144044, "all_evidence_in_top3": 0.43426518468620184, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.3715, "hop_accuracy": 0.7322992700729927, "hop_accuracy_answerable": 0.9010890310991355, "hop_under_prediction": 0.046817110138093636, "hop_over_prediction": 0.05209385876277085, "mean_score_margin": -9.29892828391031, "median_score_margin": -9.269989609718323, "negative_margin_rate": 0.5887504210171774, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 4042, "true_0_pred_2": 64, "true_1_pred_0": 834, "true_1_pred_1": 14052, "true_1_pred_2": 928, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9762653562039442, "unknown_mean_need_prob": 0.998855464207621, "thr0.30": { "need_precision": 0.8126824817518248, "need_recall": 1.0, "need_f1": 0.8966628076710123, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8126824817518248, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 17814, "tn": 0, "fp": 4106, "fn": 0 } }, "thr0.40": { "need_precision": 0.8126824817518248, "need_recall": 1.0, "need_f1": 0.8966628076710123, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8126824817518248, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 17814, "tn": 0, "fp": 4106, "fn": 0 } }, "thr0.50": { "need_precision": 0.8124086257309941, "need_recall": 0.998203660042663, "need_f1": 0.895773512669387, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8112226277372263, "known_question_refusal_rate": 0.001796339957336926, "unknown_question_read_rate": 1.0, "counts": { "tp": 17782, "tn": 0, "fp": 4106, "fn": 32 } }, "thr0.60": { "need_precision": 0.8115476409032495, "need_recall": 0.9925900976759852, "need_f1": 0.892985202767537, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8066605839416059, "known_question_refusal_rate": 0.00740990232401482, "unknown_question_read_rate": 1.0, "counts": { "tp": 17682, "tn": 0, "fp": 4106, "fn": 132 } }, "thr0.70": { "need_precision": 0.8112270700197692, "need_recall": 0.9905130796003143, "need_f1": 0.8919499557689878, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8049726277372263, "known_question_refusal_rate": 0.00948692039968564, "unknown_question_read_rate": 1.0, "counts": { "tp": 17645, "tn": 0, "fp": 4106, "fn": 169 } }, "thr0.80": { "need_precision": 0.8106525247867189, "need_recall": 0.9868081284383069, "need_f1": 0.8900984835059115, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8019616788321168, "known_question_refusal_rate": 0.01319187156169305, "unknown_question_read_rate": 1.0, "counts": { "tp": 17579, "tn": 0, "fp": 4106, "fn": 235 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_2": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.9999341862276196, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 1.0, "counts": { "tp": 0, "tn": 0, "fp": 64, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.442375, "recall1": 0.39675, "recall3": 0.49153125, "recall5": 0.50303125, "recall8": 0.51234375, "evidence_count_in_top3": 0.57475, "hit3": 0.5283125, "mrr": 0.500402578720693, "ndcg3": 0.4692792057097287, "mean_first_positive_rank": 14.9525, "all_evidence_in_top3": 0.45475, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.3715, "hop_accuracy": 0.74955, "hop_accuracy_answerable": 0.9369375, "hop_under_prediction": 0.0160625, "hop_over_prediction": 0.047, "mean_score_margin": -8.091850390457548, "median_score_margin": -6.662378132343292, "negative_margin_rate": 0.557625, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 4000, "true_0_pred_2": 0, "true_1_pred_0": 257, "true_1_pred_1": 12991, "true_1_pred_2": 752, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9821553228273988, "unknown_mean_need_prob": 0.9988265322595835, "thr0.50": { "need_precision": 0.8, "need_recall": 1.0, "need_f1": 0.888888888888889, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 16000, "tn": 0, "fp": 4000, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.14528412419449327, "recall1": 0.14528412419449327, "recall3": 0.2677211482132396, "recall5": 0.38488576449912126, "recall8": 0.5653192735793791, "evidence_count_in_top3": 0.2677211482132396, "hit3": 0.2677211482132396, "mrr": 0.2801716446629072, "ndcg3": 0.21516992510461316, "mean_first_positive_rank": 9.195664909197422, "all_evidence_in_top3": 0.2677211482132396, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.5740740740740741, "hop_accuracy_answerable": 0.5811364967779731, "hop_under_prediction": 0.3380199179847686, "hop_over_prediction": 0.08084358523725835, "mean_score_margin": -19.359866383266713, "median_score_margin": -18.159542322158813, "negative_margin_rate": 0.8547158758055068, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 21, "true_0_pred_2": 0, "true_1_pred_0": 577, "true_1_pred_1": 992, "true_1_pred_2": 138 }, "known_mean_need_prob": 0.9195722371566764, "unknown_mean_need_prob": 0.9999671209426153, "thr0.50": { "need_precision": 0.9876179245283019, "need_recall": 0.981253661394259, "need_f1": 0.9844255069056715, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.9693287037037037, "known_question_refusal_rate": 0.018746338605741066, "unknown_question_read_rate": 1.0, "counts": { "tp": 1675, "tn": 0, "fp": 21, "fn": 32 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.0, "recall1": 0.0, "recall3": 0.028037383177570093, "recall5": 0.07476635514018691, "recall8": 0.2803738317757009, "evidence_count_in_top3": 0.028037383177570093, "hit3": 0.028037383177570093, "mrr": 0.1023068389893556, "ndcg3": 0.01524233414552764, "mean_first_positive_rank": 12.598130841121495, "all_evidence_in_top3": 0.028037383177570093, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.5390625, "hop_accuracy_answerable": 0.6448598130841121, "hop_under_prediction": 0.0, "hop_over_prediction": 0.35514018691588783, "mean_score_margin": -29.291684916086286, "median_score_margin": -29.07153034210205, "negative_margin_rate": 1.0, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 21, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 69, "true_1_pred_2": 38 }, "known_mean_need_prob": 0.9999633771236812, "unknown_mean_need_prob": 0.9999671209426153, "thr0.50": { "need_precision": 0.8359375, "need_recall": 1.0, "need_f1": 0.9106382978723404, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8359375, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 107, "tn": 0, "fp": 21, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.796, "recall1": 0.796, "recall3": 0.9125, "recall5": 0.919, "recall8": 0.927, "evidence_count_in_top3": 0.9125, "hit3": 0.9125, "mrr": 0.8582544951434297, "ndcg3": 0.8678012294946459, "mean_first_positive_rank": 2.8275, "all_evidence_in_top3": 0.9125, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.4955, "hop_accuracy_answerable": 0.4955, "hop_under_prediction": 0.1285, "hop_over_prediction": 0.376, "mean_score_margin": 21.036133953779935, "median_score_margin": 21.400100469589233, "negative_margin_rate": 0.204, "hop_confusion": { "true_1_pred_0": 257, "true_1_pred_1": 991, "true_1_pred_2": 752 }, "known_mean_need_prob": 0.9764632563889026, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.231, "recall1": 0.231, "recall3": 0.2535, "recall5": 0.255, "recall8": 0.257, "evidence_count_in_top3": 0.2535, "hit3": 0.2535, "mrr": 0.26603145346142226, "ndcg3": 0.24486859507142916, "mean_first_positive_rank": 23.789, "all_evidence_in_top3": 0.2535, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -27.718347060799598, "median_score_margin": -38.59446668624878, "negative_margin_rate": 0.769, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9723067943453789, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.2255, "recall1": 0.2255, "recall3": 0.2455, "recall5": 0.2475, "recall8": 0.2485, "evidence_count_in_top3": 0.2455, "hit3": 0.2455, "mrr": 0.2596338049221357, "ndcg3": 0.23792220044107196, "mean_first_positive_rank": 24.0695, "all_evidence_in_top3": 0.2455, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -28.903205164782705, "median_score_margin": -39.01692509651184, "negative_margin_rate": 0.7745, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9724746745526791, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.99774328109622, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 1.0, "counts": { "tp": 0, "tn": 0, "fp": 2000, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.5255, "recall1": 0.5255, "recall3": 0.6245, "recall5": 0.6265, "recall8": 0.6295, "evidence_count_in_top3": 0.6245, "hit3": 0.6245, "mrr": 0.5855216777342396, "ndcg3": 0.585867169546431, "mean_first_positive_rank": 12.2085, "all_evidence_in_top3": 0.6245, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -1.0039909677058458, "median_score_margin": 2.929476022720337, "negative_margin_rate": 0.4745, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9875293919146061, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.73, "recall1": 0.365, "recall3": 0.66575, "recall5": 0.72725, "recall8": 0.77375, "evidence_count_in_top3": 1.3315, "hit3": 0.96, "mrr": 0.84532448592014, "ndcg3": 0.6658792959224611, "mean_first_positive_rank": 1.59, "all_evidence_in_top3": 0.3715, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.3715, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.786984693795443, "median_score_margin": 5.27826452255249, "negative_margin_rate": 0.27, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9993210740983486, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.443, "recall1": 0.443, "recall3": 0.5525, "recall5": 0.5615, "recall8": 0.5665, "evidence_count_in_top3": 0.5525, "hit3": 0.5525, "mrr": 0.5118924660255056, "ndcg3": 0.5092063534375025, "mean_first_positive_rank": 13.8255, "all_evidence_in_top3": 0.5525, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -2.995655853047967, "median_score_margin": -5.5251545906066895, "negative_margin_rate": 0.557, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9880377069413662, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.249, "recall1": 0.249, "recall3": 0.287, "recall5": 0.293, "recall8": 0.3005, "evidence_count_in_top3": 0.287, "hit3": 0.287, "mrr": 0.2919631936667751, "ndcg3": 0.27173149797678653, "mean_first_positive_rank": 22.067, "all_evidence_in_top3": 0.287, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -19.72415616109967, "median_score_margin": -21.928287506103516, "negative_margin_rate": 0.751, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9656237106621265, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.339, "recall1": 0.339, "recall3": 0.391, "recall5": 0.3945, "recall8": 0.396, "evidence_count_in_top3": 0.391, "hit3": 0.391, "mrr": 0.3845990528918967, "ndcg3": 0.3709573037875013, "mean_first_positive_rank": 19.243, "all_evidence_in_top3": 0.391, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -8.212566563799978, "median_score_margin": -16.937832549214363, "negative_margin_rate": 0.661, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9954859737157822, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.13671444321940462, "recall1": 0.13671444321940462, "recall3": 0.2535832414553473, "recall5": 0.36659316427783906, "recall8": 0.5485115766262404, "evidence_count_in_top3": 0.2535832414553473, "hit3": 0.2535832414553473, "mrr": 0.26968017045834825, "ndcg3": 0.2033770627933551, "mean_first_positive_rank": 9.396361631753033, "all_evidence_in_top3": 0.2535832414553473, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.5526041666666667, "hop_accuracy_answerable": 0.5848952590959207, "hop_under_prediction": 0.31808158765159866, "hop_over_prediction": 0.0970231532524807, "mean_score_margin": -19.945701324287494, "median_score_margin": -18.862584590911865, "negative_margin_rate": 0.8632855567805954, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 42, "true_0_pred_2": 64, "true_1_pred_0": 577, "true_1_pred_1": 1061, "true_1_pred_2": 176 }, "known_mean_need_prob": 0.9243141621712682, "unknown_mean_need_prob": 0.9999472358316746, "thr0.50": { "need_precision": 0.9438559322033898, "need_recall": 0.9823594266813671, "need_f1": 0.9627228525121555, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.928125, "known_question_refusal_rate": 0.017640573318632856, "unknown_question_read_rate": 1.0, "counts": { "tp": 1782, "tn": 0, "fp": 106, "fn": 32 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.9999097834229469, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 1.0, "counts": { "tp": 0, "tn": 0, "fp": 2000, "fn": 0 } } } } }, "V2-128 v6 final": { "checkpoint": "checkpoints\\router_v6_v2_128\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 8156921 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 8156921 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.9277419995729966, "single_query_latency_ms_p50": 0.9184999944409356, "single_query_latency_ms_p95": 1.146100003097672, "queries_per_second": 1077.8858782509155, "samples": 150, "batched_qps_64": 72856.42735601631, "batched_latency_ms_64": 0.013725624990001961, "batched_qps_256": 236293.15117731004, "batched_latency_ms_256": 0.004232031250239743 }, "cpu": { "single_query_latency_ms_mean": 0.46528599991385516, "single_query_latency_ms_p50": 0.4394999996293336, "single_query_latency_ms_p95": 0.6572999991476536, "queries_per_second": 2149.215751570311, "samples": 150, "batched_qps_64": 10744.09578471459, "batched_latency_ms_64": 0.093074374990465, "batched_qps_256": 12172.553554284297, "batched_latency_ms_256": 0.08215203125132575 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9436959694622207, "recall1": 0.8882059054676097, "recall3": 0.9639609296059279, "recall5": 0.973981138430448, "recall8": 0.9864151790726395, "evidence_count_in_top3": 1.0717974626698104, "hit3": 0.9683956438756035, "mrr": 0.9597054333626427, "ndcg3": 0.9530820526370456, "mean_first_positive_rank": 1.2573818345121814, "all_evidence_in_top3": 0.9595262153362524, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.921, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 51.315328869764656, "median_score_margin": 56.234654903411865, "negative_margin_rate": 0.05630403053777928, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999984274064, "unknown_mean_need_prob": 4.836221294362452e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 4.8406160974828055e-06, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.9838125, "recall1": 0.92203125, "recall3": 0.9950625, "recall5": 0.99978125, "recall8": 0.999875, "evidence_count_in_top3": 1.115125, "hit3": 1.0, "mrr": 0.991875, "ndcg3": 0.9876817889701721, "mean_first_positive_rank": 1.016375, "all_evidence_in_top3": 0.990125, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.921, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 57.110110210259094, "median_score_margin": 59.90444374084473, "negative_margin_rate": 0.0161875, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999701977, "unknown_mean_need_prob": 6.379481702447893e-11, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.5940246045694201, "recall1": 0.5940246045694201, "recall3": 0.6842413591095489, "recall5": 0.7352079671939075, "recall8": 0.8611599297012302, "evidence_count_in_top3": 0.6842413591095489, "hit3": 0.6842413591095489, "mrr": 0.6760273585431921, "ndcg3": 0.6466497456649108, "mean_first_positive_rank": 3.4463971880492092, "all_evidence_in_top3": 0.6842413591095489, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.27154229221702947, "median_score_margin": 7.975786209106445, "negative_margin_rate": 0.40597539543058, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999838679872, "unknown_mean_need_prob": 0.0004654159529796483, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.5233644859813084, "recall1": 0.5233644859813084, "recall3": 0.7757009345794392, "recall5": 0.9252336448598131, "recall8": 0.9719626168224299, "evidence_count_in_top3": 0.7757009345794392, "hit3": 0.7757009345794392, "mrr": 0.674896157840083, "ndcg3": 0.6678873486315127, "mean_first_positive_rank": 2.3738317757009346, "all_evidence_in_top3": 0.7757009345794392, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.8786688875929217, "median_score_margin": 0.4756507873535156, "negative_margin_rate": 0.4766355140186916, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0004654159529796483, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.993, "recall1": 0.993, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9965, "ndcg3": 0.9974165082750002, "mean_first_positive_rank": 1.007, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 54.4210530708991, "median_score_margin": 56.92102670669556, "negative_margin_rate": 0.007, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.975, "recall1": 0.975, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9875, "ndcg3": 0.9907732438392864, "mean_first_positive_rank": 1.025, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 64.29146421498805, "median_score_margin": 73.40720510482788, "negative_margin_rate": 0.025, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9845, "recall1": 0.9845, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99225, "ndcg3": 0.9942794111803576, "mean_first_positive_rank": 1.0155, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 65.52974343589041, "median_score_margin": 73.97303533554077, "negative_margin_rate": 0.0155, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.275862421441728e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9795, "recall1": 0.9795, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9896666666666666, "ndcg3": 0.9923685950714292, "mean_first_positive_rank": 1.021, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 60.150181606799364, "median_score_margin": 68.01980018615723, "negative_margin_rate": 0.0205, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9885, "recall1": 0.49425, "recall3": 0.9605, "recall5": 0.99825, "recall8": 0.999, "evidence_count_in_top3": 1.921, "hit3": 1.0, "mrr": 0.99425, "ndcg3": 0.9452009954703022, "mean_first_positive_rank": 1.0115, "all_evidence_in_top3": 0.921, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.921, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 55.6439196386747, "median_score_margin": 62.50904202461243, "negative_margin_rate": 0.0115, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999997615814, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.995, "recall1": 0.995, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9975, "ndcg3": 0.9981546487678573, "mean_first_positive_rank": 1.005, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 48.13904330630228, "median_score_margin": 49.25835180282593, "negative_margin_rate": 0.005, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9805, "recall1": 0.9805, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9901666666666666, "ndcg3": 0.9927376653178577, "mean_first_positive_rank": 1.02, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 50.85701001610607, "median_score_margin": 54.9089093208313, "negative_margin_rate": 0.0195, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9745, "recall1": 0.9745, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9871666666666666, "ndcg3": 0.9905232438392865, "mean_first_positive_rank": 1.026, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 57.84846639241278, "median_score_margin": 63.093711376190186, "negative_margin_rate": 0.0255, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.5898566703417861, "recall1": 0.5898566703417861, "recall3": 0.6896361631753032, "recall5": 0.7464167585446527, "recall8": 0.8676957001102535, "evidence_count_in_top3": 0.6896361631753032, "hit3": 0.6896361631753032, "mrr": 0.6759606339151697, "ndcg3": 0.6479024598421029, "mean_first_positive_rank": 3.383131201764057, "all_evidence_in_top3": 0.6896361631753032, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.20369631854576997, "median_score_margin": 6.827788352966309, "negative_margin_rate": 0.4101433296582139, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999848195447, "unknown_mean_need_prob": 0.00018733273071117103, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 3.3919047850488803e-15, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "REPLAY-128 v7 final": { "checkpoint": "checkpoints\\router_replay_v7_v2_128\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 8156921 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 8156921 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.9663366669459114, "single_query_latency_ms_p50": 0.9168999968096614, "single_query_latency_ms_p95": 1.2540000025182962, "queries_per_second": 1034.8360299319709, "samples": 150, "batched_qps_64": 69378.19790068052, "batched_latency_ms_64": 0.014413750000130676, "batched_qps_256": 242759.87633474605, "batched_latency_ms_256": 0.004119296875160217 }, "cpu": { "single_query_latency_ms_mean": 0.3868726668103288, "single_query_latency_ms_p50": 0.36950000503566116, "single_query_latency_ms_p95": 0.5363999953260645, "queries_per_second": 2584.829805229605, "samples": 150, "batched_qps_64": 13315.579227047761, "batched_latency_ms_64": 0.07510000000365835, "batched_qps_256": 12736.711281807726, "batched_latency_ms_256": 0.07851320312397547 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9413944088918828, "recall1": 0.8857920736499383, "recall3": 0.9647748961490962, "recall5": 0.9714550353654429, "recall8": 0.9804086673402942, "evidence_count_in_top3": 1.0744919726058157, "hit3": 0.9673290670259347, "mrr": 0.957689277532011, "ndcg3": 0.9536787031890833, "mean_first_positive_rank": 1.34001347254968, "all_evidence_in_top3": 0.9622207252722578, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9545, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 40.14557690149532, "median_score_margin": 45.34479379653931, "negative_margin_rate": 0.05860559110811721, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999984274064, "unknown_mean_need_prob": 4.836221294362452e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 4.8406160974828055e-06, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.98225, "recall1": 0.92034375, "recall3": 0.99715625, "recall5": 0.99959375, "recall8": 0.9996875, "evidence_count_in_top3": 1.1193125, "hit3": 1.0, "mrr": 0.9911041666666667, "ndcg3": 0.9893498205729669, "mean_first_positive_rank": 1.017875, "all_evidence_in_top3": 0.9943125, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9545, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 44.73340179644042, "median_score_margin": 47.92497134208679, "negative_margin_rate": 0.01775, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999701977, "unknown_mean_need_prob": 6.379481702447893e-11, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.5875805506736965, "recall1": 0.5875805506736965, "recall3": 0.6731107205623902, "recall5": 0.7106033977738723, "recall8": 0.7996485061511424, "evidence_count_in_top3": 0.6731107205623902, "hit3": 0.6731107205623902, "mrr": 0.6633131807141506, "ndcg3": 0.6380925044585338, "mean_first_positive_rank": 4.295254833040421, "all_evidence_in_top3": 0.6731107205623902, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.2721888081208264, "median_score_margin": 8.391286849975586, "negative_margin_rate": 0.4124194493263035, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999838679872, "unknown_mean_need_prob": 0.0004654159529796483, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.4766355140186916, "recall1": 0.4766355140186916, "recall3": 0.7757009345794392, "recall5": 0.9252336448598131, "recall8": 0.9813084112149533, "evidence_count_in_top3": 0.7757009345794392, "hit3": 0.7757009345794392, "mrr": 0.6573319982198488, "ndcg3": 0.6543120031041451, "mean_first_positive_rank": 2.364485981308411, "all_evidence_in_top3": 0.7757009345794392, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -1.0887432181946586, "median_score_margin": -0.6999244689941406, "negative_margin_rate": 0.5233644859813084, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0004654159529796483, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.991, "recall1": 0.991, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9955, "ndcg3": 0.9966783677821431, "mean_first_positive_rank": 1.009, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 42.24227243242413, "median_score_margin": 42.77137851715088, "negative_margin_rate": 0.009, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.973, "recall1": 0.973, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9865, "ndcg3": 0.9900351033464293, "mean_first_positive_rank": 1.027, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 50.82861385926604, "median_score_margin": 54.98523235321045, "negative_margin_rate": 0.027, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.983, "recall1": 0.983, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9915, "ndcg3": 0.9937258058107148, "mean_first_positive_rank": 1.017, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 51.79758185644448, "median_score_margin": 55.65491724014282, "negative_margin_rate": 0.017, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.275862421441728e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.978, "recall1": 0.978, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9889166666666667, "ndcg3": 0.9918149897017863, "mean_first_positive_rank": 1.0225, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 47.034401270009575, "median_score_margin": 50.28937745094299, "negative_margin_rate": 0.022, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9905, "recall1": 0.49525, "recall3": 0.97725, "recall5": 0.99675, "recall8": 0.9975, "evidence_count_in_top3": 1.9545, "hit3": 1.0, "mrr": 0.99525, "ndcg3": 0.963831301989089, "mean_first_positive_rank": 1.0095, "all_evidence_in_top3": 0.9545, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9545, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 46.103771095309405, "median_score_margin": 52.25391101837158, "negative_margin_rate": 0.0095, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999997615814, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.994, "recall1": 0.994, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.997, "ndcg3": 0.9977855785214288, "mean_first_positive_rank": 1.006, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 36.59333083114028, "median_score_margin": 35.444119691848755, "negative_margin_rate": 0.006, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.978, "recall1": 0.978, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9889166666666667, "ndcg3": 0.9918149897017863, "mean_first_positive_rank": 1.0225, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 35.780669214252384, "median_score_margin": 35.064393758773804, "negative_margin_rate": 0.022, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9705, "recall1": 0.9705, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98525, "ndcg3": 0.989112427730358, "mean_first_positive_rank": 1.0295, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 47.48657381267706, "median_score_margin": 51.48151856660843, "negative_margin_rate": 0.0295, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.5810363836824697, "recall1": 0.5810363836824697, "recall3": 0.679162072767365, "recall5": 0.7232635060639471, "recall8": 0.8103638368246968, "evidence_count_in_top3": 0.679162072767365, "hit3": 0.679162072767365, "mrr": 0.662960376675071, "ndcg3": 0.6390492224051051, "mean_first_positive_rank": 4.181367144432194, "all_evidence_in_top3": 0.679162072767365, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.32035381466873164, "median_score_margin": 7.441610336303711, "negative_margin_rate": 0.4189636163175303, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999848195447, "unknown_mean_need_prob": 0.00018733273071117103, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 3.3919047850488803e-15, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "V2-512 v6 final": { "checkpoint": "checkpoints\\router_v6_v2_512\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3 }, "parameters": 4741902, "checkpoint_bytes": 18973433 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048, "checkpoint_bytes": 18973433 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.9488446664181538, "single_query_latency_ms_p50": 0.8728999964660034, "single_query_latency_ms_p95": 1.1364000019966625, "queries_per_second": 1053.9132856961248, "samples": 150, "batched_qps_64": 71586.76538686478, "batched_latency_ms_64": 0.01396906250192842, "batched_qps_256": 177958.2077420389, "batched_latency_ms_256": 0.005619296871373081 }, "cpu": { "single_query_latency_ms_mean": 0.8265033330826554, "single_query_latency_ms_p50": 0.8115000018733554, "single_query_latency_ms_p95": 1.039499999023974, "queries_per_second": 1209.9164757996127, "samples": 150, "batched_qps_64": 3171.5826196153826, "batched_latency_ms_64": 0.3153000000111206, "batched_qps_256": 3134.2351278810493, "batched_latency_ms_256": 0.3190571093739436 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.941843493881217, "recall1": 0.8864657011339396, "recall3": 0.9666273717300999, "recall5": 0.9764230380599529, "recall8": 0.984506567867969, "evidence_count_in_top3": 1.07668126192882, "hit3": 0.9688447288649377, "mrr": 0.958724954839147, "ndcg3": 0.9548399733910118, "mean_first_positive_rank": 1.2825867295385651, "all_evidence_in_top3": 0.9644100145952621, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9605, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 73.24820388849786, "median_score_margin": 76.59281730651855, "negative_margin_rate": 0.05815650611878298, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999984274064, "unknown_mean_need_prob": 4.522551017238816e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 4.432710963581804e-06, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.98375, "recall1": 0.92209375, "recall3": 0.99753125, "recall5": 0.9999375, "recall8": 1.0, "evidence_count_in_top3": 1.1200625, "hit3": 1.0, "mrr": 0.99184375, "ndcg3": 0.9902417180168516, "mean_first_positive_rank": 1.0164375, "all_evidence_in_top3": 0.9950625, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9605, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 81.85253369444608, "median_score_margin": 83.23931503295898, "negative_margin_rate": 0.01625, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999701977, "unknown_mean_need_prob": 6.058075942262249e-11, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.5770357352079672, "recall1": 0.5770357352079672, "recall3": 0.6859988283538372, "recall5": 0.7562975981253661, "recall8": 0.8388986526069128, "evidence_count_in_top3": 0.6859988283538372, "hit3": 0.6859988283538372, "mrr": 0.6663638286240626, "ndcg3": 0.6401845706816489, "mean_first_positive_rank": 3.719976567076743, "all_evidence_in_top3": 0.6859988283538372, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -2.648404171182946, "median_score_margin": 5.544002532958984, "negative_margin_rate": 0.4229642647920328, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999838679872, "unknown_mean_need_prob": 0.00043537282504942034, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.4953271028037383, "recall1": 0.4953271028037383, "recall3": 0.822429906542056, "recall5": 0.9719626168224299, "recall8": 0.9906542056074766, "evidence_count_in_top3": 0.822429906542056, "hit3": 0.822429906542056, "mrr": 0.6704980377877574, "ndcg3": 0.680904070694264, "mean_first_positive_rank": 2.196261682242991, "all_evidence_in_top3": 0.822429906542056, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -2.581393656329574, "median_score_margin": -0.056890010833740234, "negative_margin_rate": 0.5046728971962616, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.00043537282504942034, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9935, "recall1": 0.9935, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99675, "ndcg3": 0.9976010433982145, "mean_first_positive_rank": 1.0065, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 64.64610423742235, "median_score_margin": 60.52951920032501, "negative_margin_rate": 0.0065, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.98, "recall1": 0.98, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99, "ndcg3": 0.9926185950714291, "mean_first_positive_rank": 1.02, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 88.94486605204642, "median_score_margin": 95.25923618674278, "negative_margin_rate": 0.02, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9805, "recall1": 0.9805, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9901666666666666, "ndcg3": 0.9927376653178577, "mean_first_positive_rank": 1.02, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 90.02319332621246, "median_score_margin": 96.11362898349762, "negative_margin_rate": 0.0195, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.2115745555928435e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.977, "recall1": 0.977, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9884166666666666, "ndcg3": 0.9914459194553578, "mean_first_positive_rank": 1.0235, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 78.39938361007721, "median_score_margin": 81.58803009986877, "negative_margin_rate": 0.023, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9865, "recall1": 0.49325, "recall3": 0.98025, "recall5": 0.9995, "recall8": 1.0, "evidence_count_in_top3": 1.9605, "hit3": 1.0, "mrr": 0.99325, "ndcg3": 0.9651268224740949, "mean_first_positive_rank": 1.0135, "all_evidence_in_top3": 0.9605, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9605, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 104.36254599650204, "median_score_margin": 118.10550880432129, "negative_margin_rate": 0.0135, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999997615814, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9955, "recall1": 0.9955, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99775, "ndcg3": 0.9983391838910716, "mean_first_positive_rank": 1.0045, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 57.61693591082841, "median_score_margin": 57.403422355651855, "negative_margin_rate": 0.0045, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9825, "recall1": 0.9825, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99125, "ndcg3": 0.9935412706875005, "mean_first_positive_rank": 1.0175, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 84.6615379076004, "median_score_margin": 90.02162456512451, "negative_margin_rate": 0.0175, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9745, "recall1": 0.9745, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9871666666666666, "ndcg3": 0.9905232438392865, "mean_first_positive_rank": 1.026, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 86.1657025148794, "median_score_margin": 91.7922420501709, "negative_margin_rate": 0.0255, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.5722160970231532, "recall1": 0.5722160970231532, "recall3": 0.6940463065049615, "recall5": 0.7690187431091511, "recall8": 0.8478500551267916, "evidence_count_in_top3": 0.6940463065049615, "hit3": 0.6940463065049615, "mrr": 0.666607687709242, "ndcg3": 0.6425864375511913, "mean_first_positive_rank": 3.630099228224917, "all_evidence_in_top3": 0.6940463065049615, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -2.644451511266016, "median_score_margin": 4.475192725658417, "negative_margin_rate": 0.42778390297684676, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999848195447, "unknown_mean_need_prob": 0.00017518256748815932, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 4.063285960620704e-15, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "XL-512 v6 final": { "checkpoint": "checkpoints\\router_v6_xl512\\memory_router_xl.pt", "router": { "kind": "router_xl", "step": null, "arch_config": { "arch": "router_xl", "arch_version": 1, "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3, "encoder_layers": 2, "encoder_hidden": 512, "pair_blocks": 1, "pair_hidden": 512, "pair_expansion": 2, "pair_dropout": 0.05, "use_interaction": true, "policy_layers": 2, "policy_hidden": 512, "policy_dropout": 0.0, "learnable_cosine_scale": true }, "parameters": 7898127, "checkpoint_bytes": 35802155 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048, "checkpoint_bytes": 35802155 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.450062666942055, "single_query_latency_ms_p50": 1.4245999991544522, "single_query_latency_ms_p95": 1.9877999948221259, "queries_per_second": 689.6253677841637, "samples": 150, "batched_qps_64": 46891.9433921584, "batched_latency_ms_64": 0.021325624993551173, "batched_qps_256": 85734.57110724591, "batched_latency_ms_256": 0.011663906252579181 }, "cpu": { "single_query_latency_ms_mean": 2.099922667063462, "single_query_latency_ms_p50": 1.9889000031980686, "single_query_latency_ms_p95": 2.6856999975279905, "queries_per_second": 476.2080126495338, "samples": 150, "batched_qps_64": 1630.631411069136, "batched_latency_ms_64": 0.613259374995323, "batched_qps_256": 1668.1377991342565, "batched_latency_ms_256": 0.5994708593732412 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9502077018075671, "recall1": 0.8948299090602897, "recall3": 0.965336252385764, "recall5": 0.9760862243179522, "recall8": 0.9854608734703043, "evidence_count_in_top3": 1.0657348153137982, "hit3": 0.9772089367912877, "mrr": 0.9664213223997663, "ndcg3": 0.9560053718886551, "mean_first_positive_rank": 1.1716066015493432, "all_evidence_in_top3": 0.9534635679802402, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.7885, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 215.75331382084113, "median_score_margin": 239.68688774108887, "negative_margin_rate": 0.049792298192432916, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999979790499, "unknown_mean_need_prob": 4.1244088529706584e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 4.264552800687521e-06, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.984, "recall1": 0.92234375, "recall3": 0.98678125, "recall5": 0.990875, "recall8": 0.9933125, "evidence_count_in_top3": 1.0985625, "hit3": 1.0, "mrr": 0.99196875, "ndcg3": 0.9823026449115756, "mean_first_positive_rank": 1.0161875, "all_evidence_in_top3": 0.9735625, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.7885, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 235.8910635419197, "median_score_margin": 250.7837634086609, "negative_margin_rate": 0.016, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999627471, "unknown_mean_need_prob": 7.011441629130955e-11, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.6690099589923842, "recall1": 0.6690099589923842, "recall3": 0.7709431751611013, "recall5": 0.8388986526069128, "recall8": 0.9121265377855887, "evidence_count_in_top3": 0.7709431751611013, "hit3": 0.7709431751611013, "mrr": 0.7488416468644941, "ndcg3": 0.7281836459473614, "mean_first_positive_rank": 2.558289396602226, "all_evidence_in_top3": 0.7709431751611013, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 40.85260479970152, "median_score_margin": 62.14678978919983, "negative_margin_rate": 0.3309900410076157, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999792588407, "unknown_mean_need_prob": 0.00039670502174734185, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.38317757009345793, "recall1": 0.38317757009345793, "recall3": 0.8598130841121495, "recall5": 0.9532710280373832, "recall8": 0.9813084112149533, "evidence_count_in_top3": 0.8598130841121495, "hit3": 0.8598130841121495, "mrr": 0.617352766651832, "ndcg3": 0.6582046038050816, "mean_first_positive_rank": 2.289719626168224, "all_evidence_in_top3": 0.8598130841121495, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -5.260566909736562, "median_score_margin": -7.525552272796631, "negative_margin_rate": 0.616822429906542, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.00039670502174734185, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9925, "recall1": 0.9925, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99625, "ndcg3": 0.9972319731517859, "mean_first_positive_rank": 1.0075, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 272.5490972074904, "median_score_margin": 288.63626432418823, "negative_margin_rate": 0.0075, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.98, "recall1": 0.98, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99, "ndcg3": 0.9926185950714291, "mean_first_positive_rank": 1.02, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 215.3791910160221, "median_score_margin": 239.17540168762207, "negative_margin_rate": 0.02, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.982, "recall1": 0.982, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9909166666666667, "ndcg3": 0.9932912706875006, "mean_first_positive_rank": 1.0185, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 219.1182684260085, "median_score_margin": 239.48388767242432, "negative_margin_rate": 0.018, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.4022628460683615e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.983, "recall1": 0.983, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9914166666666666, "ndcg3": 0.9936603409339291, "mean_first_positive_rank": 1.0175, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 211.44927242165804, "median_score_margin": 233.2896122932434, "negative_margin_rate": 0.017, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9865, "recall1": 0.49325, "recall3": 0.89425, "recall5": 0.927, "recall8": 0.9465, "evidence_count_in_top3": 1.7885, "hit3": 1.0, "mrr": 0.99325, "ndcg3": 0.9008760971390303, "mean_first_positive_rank": 1.0135, "all_evidence_in_top3": 0.7885, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.7885, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 274.43398497897016, "median_score_margin": 301.56740951538086, "negative_margin_rate": 0.0135, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999997019767, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.995, "recall1": 0.995, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9975, "ndcg3": 0.9981546487678573, "mean_first_positive_rank": 1.005, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 223.85322645658067, "median_score_margin": 222.2765736579895, "negative_margin_rate": 0.005, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9805, "recall1": 0.9805, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99025, "ndcg3": 0.9928031301946435, "mean_first_positive_rank": 1.0195, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 225.64811652239038, "median_score_margin": 241.96742022037506, "negative_margin_rate": 0.0195, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9725, "recall1": 0.9725, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9861666666666666, "ndcg3": 0.9897851033464293, "mean_first_positive_rank": 1.028, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 244.6973513062373, "median_score_margin": 263.6290616989136, "negative_margin_rate": 0.0275, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.6521499448732084, "recall1": 0.6521499448732084, "recall3": 0.7761852260198456, "recall5": 0.8456449834619625, "recall8": 0.9162072767364939, "evidence_count_in_top3": 0.7761852260198456, "hit3": 0.7761852260198456, "mrr": 0.7410856875575729, "ndcg3": 0.7240558854681862, "mean_first_positive_rank": 2.54244762954796, "all_evidence_in_top3": 0.7761852260198456, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 38.1325886073587, "median_score_margin": 54.42159175872803, "negative_margin_rate": 0.3478500551267916, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999804822718, "unknown_mean_need_prob": 0.0001597598329493619, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 2.5479757829358922e-15, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "XL-128 v6 final": { "checkpoint": "checkpoints\\router_v6_xl128\\memory_router_xl.pt", "router": { "kind": "router_xl", "step": null, "arch_config": { "arch": "router_xl", "arch_version": 1, "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3, "encoder_layers": 2, "encoder_hidden": 512, "pair_blocks": 1, "pair_hidden": 512, "pair_expansion": 2, "pair_dropout": 0.05, "use_interaction": true, "policy_layers": 2, "policy_hidden": 512, "policy_dropout": 0.0, "learnable_cosine_scale": true }, "parameters": 6714639, "checkpoint_bytes": 27922475 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 27922475 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.5358359999663662, "single_query_latency_ms_p50": 1.4220000011846423, "single_query_latency_ms_p95": 2.4550000016461127, "queries_per_second": 651.1111863648849, "samples": 150, "batched_qps_64": 33111.55489890671, "batched_latency_ms_64": 0.03020093749910302, "batched_qps_256": 131545.14154150538, "batched_latency_ms_256": 0.007601953126368244 }, "cpu": { "single_query_latency_ms_mean": 1.8319966668301884, "single_query_latency_ms_p50": 1.8206000022473745, "single_query_latency_ms_p95": 2.2160000007716008, "queries_per_second": 545.8525215170013, "samples": 150, "batched_qps_64": 2173.5896629680874, "batched_latency_ms_64": 0.4600684374963748, "batched_qps_256": 2384.8277260004847, "batched_latency_ms_256": 0.41931750000117063 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9518917705175705, "recall1": 0.8963736387111261, "recall3": 0.9745424946671157, "recall5": 0.9820366004266308, "recall8": 0.9912709105198159, "evidence_count_in_top3": 1.0851577411025037, "hit3": 0.9761984955652857, "mrr": 0.9670023550641874, "ndcg3": 0.9642642772784563, "mean_first_positive_rank": 1.1905243067250477, "all_evidence_in_top3": 0.9728864937689458, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9705, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 200.33457717921095, "median_score_margin": 219.4529800415039, "negative_margin_rate": 0.04810822948242955, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999979790499, "unknown_mean_need_prob": 3.8975788264237145e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 3.895038815020929e-06, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.984375, "recall1": 0.9225625, "recall3": 0.99815625, "recall5": 0.9995, "recall8": 0.99959375, "evidence_count_in_top3": 1.1213125, "hit3": 1.0, "mrr": 0.9921354166666666, "ndcg3": 0.9914167841247224, "mean_first_positive_rank": 1.0159375, "all_evidence_in_top3": 0.9963125, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9705, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 217.4091286659348, "median_score_margin": 228.10742282867432, "negative_margin_rate": 0.015625, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999627471, "unknown_mean_need_prob": 6.477803048274694e-11, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.6754540128881078, "recall1": 0.6754540128881078, "recall3": 0.7592267135325131, "recall5": 0.8189806678383128, "recall8": 0.9127123608670181, "evidence_count_in_top3": 0.7592267135325131, "hit3": 0.7592267135325131, "mrr": 0.7481470978152184, "ndcg3": 0.7247037236923608, "mean_first_positive_rank": 2.7809021675454013, "all_evidence_in_top3": 0.7592267135325131, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 52.43232965865677, "median_score_margin": 72.44926834106445, "negative_margin_rate": 0.3245459871118922, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999792588407, "unknown_mean_need_prob": 0.0003750932634526786, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.5046728971962616, "recall1": 0.5046728971962616, "recall3": 0.8785046728971962, "recall5": 0.9719626168224299, "recall8": 1.0, "evidence_count_in_top3": 0.8785046728971962, "hit3": 0.8785046728971962, "mrr": 0.7002447708055184, "ndcg3": 0.7258507766355217, "mean_first_positive_rank": 1.925233644859813, "all_evidence_in_top3": 0.8785046728971962, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 6.655256898882233, "median_score_margin": 1.441606044769287, "negative_margin_rate": 0.4953271028037383, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0003750932634526786, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9945, "recall1": 0.9945, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99725, "ndcg3": 0.9979701136446429, "mean_first_positive_rank": 1.0055, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 243.0819873433709, "median_score_margin": 235.45626258850098, "negative_margin_rate": 0.0055, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9785, "recall1": 0.9785, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98925, "ndcg3": 0.9920649897017864, "mean_first_positive_rank": 1.0215, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 229.82924858646467, "median_score_margin": 248.98048400878906, "negative_margin_rate": 0.0215, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9845, "recall1": 0.9845, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9921666666666666, "ndcg3": 0.9942139463035718, "mean_first_positive_rank": 1.016, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 232.02638172297924, "median_score_margin": 249.2184543609619, "negative_margin_rate": 0.0155, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.2955399044485864e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9815, "recall1": 0.9815, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9906666666666666, "ndcg3": 0.9931067355642863, "mean_first_positive_rank": 1.019, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 211.99539810842276, "median_score_margin": 225.90464401245117, "negative_margin_rate": 0.0185, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.989, "recall1": 0.4945, "recall3": 0.98525, "recall5": 0.996, "recall8": 0.99675, "evidence_count_in_top3": 1.9705, "hit3": 1.0, "mrr": 0.9945, "ndcg3": 0.9737356054745618, "mean_first_positive_rank": 1.011, "all_evidence_in_top3": 0.9705, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9705, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 234.29880876231567, "median_score_margin": 254.67290210723877, "negative_margin_rate": 0.011, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999997019767, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.997, "recall1": 0.997, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9985, "ndcg3": 0.9988927892607143, "mean_first_positive_rank": 1.003, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 155.67033711057155, "median_score_margin": 155.824444770813, "negative_margin_rate": 0.003, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.981, "recall1": 0.981, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9905, "ndcg3": 0.9929876653178578, "mean_first_positive_rank": 1.019, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 217.68418576198818, "median_score_margin": 236.54215621948242, "negative_margin_rate": 0.019, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.969, "recall1": 0.969, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98425, "ndcg3": 0.988362427730358, "mean_first_positive_rank": 1.0325, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 214.68668193136529, "median_score_margin": 221.31105995178223, "negative_margin_rate": 0.031, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.665380374862183, "recall1": 0.665380374862183, "recall3": 0.7662624035281147, "recall5": 0.8280044101433297, "recall8": 0.9178610804851157, "evidence_count_in_top3": 0.7662624035281147, "hit3": 0.7662624035281147, "mrr": 0.745321547104062, "ndcg3": 0.7247713833753366, "mean_first_positive_rank": 2.7304299889746417, "all_evidence_in_top3": 0.7662624035281147, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 49.73213848704934, "median_score_margin": 64.13029479980469, "negative_margin_rate": 0.33461962513781696, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999804822718, "unknown_mean_need_prob": 0.0001509735806525834, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 2.0705206352511507e-15, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } } }