{ "V2-128 deployed(v3)": { "checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_qwen_router_entities\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 8157561 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 8157561 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.9698253340320662, "single_query_latency_ms_p50": 0.8987999899545684, "single_query_latency_ms_p95": 1.46159999712836, "queries_per_second": 1031.1135056067076, "samples": 150, "batched_qps_64": 64355.240952926746, "batched_latency_ms_64": 0.015538749994448153, "batched_qps_256": 185281.68609572947, "batched_latency_ms_256": 0.005397187499056599 }, "cpu": { "single_query_latency_ms_mean": 0.47286199949060875, "single_query_latency_ms_p50": 0.4398000019136816, "single_query_latency_ms_p95": 0.7371999963652343, "queries_per_second": 2114.7819048205424, "samples": 150, "batched_qps_64": 11860.681466374921, "batched_latency_ms_64": 0.08431218752775749, "batched_qps_256": 12256.36876291884, "batched_latency_ms_256": 0.08159023437883661 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.4112495789828225, "recall1": 0.37027057370607386, "recall3": 0.4673009992141013, "recall5": 0.4891377568204783, "recall8": 0.5160267205568654, "evidence_count_in_top3": 0.5420455821264174, "hit3": 0.5003368137420007, "mrr": 0.476908110965675, "ndcg3": 0.44220238482445295, "mean_first_positive_rank": 14.38671831144044, "all_evidence_in_top3": 0.43426518468620184, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.3715, "hop_accuracy": 0.7322992700729927, "hop_accuracy_answerable": 0.9010890310991355, "hop_under_prediction": 0.046817110138093636, "hop_over_prediction": 0.05209385876277085, "mean_score_margin": -9.29892828391031, "median_score_margin": -9.269989609718323, "negative_margin_rate": 0.5887504210171774, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 4042, "true_0_pred_2": 64, "true_1_pred_0": 834, "true_1_pred_1": 14052, "true_1_pred_2": 928, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9762653562039442, "unknown_mean_need_prob": 0.998855464207621, "thr0.30": { "need_precision": 0.8126824817518248, "need_recall": 1.0, "need_f1": 0.8966628076710123, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8126824817518248, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 17814, "tn": 0, "fp": 4106, "fn": 0 } }, "thr0.40": { "need_precision": 0.8126824817518248, "need_recall": 1.0, "need_f1": 0.8966628076710123, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8126824817518248, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 17814, "tn": 0, "fp": 4106, "fn": 0 } }, "thr0.50": { "need_precision": 0.8124086257309941, "need_recall": 0.998203660042663, "need_f1": 0.895773512669387, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8112226277372263, "known_question_refusal_rate": 0.001796339957336926, "unknown_question_read_rate": 1.0, "counts": { "tp": 17782, "tn": 0, "fp": 4106, "fn": 32 } }, "thr0.60": { "need_precision": 0.8115476409032495, "need_recall": 0.9925900976759852, "need_f1": 0.892985202767537, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8066605839416059, "known_question_refusal_rate": 0.00740990232401482, "unknown_question_read_rate": 1.0, "counts": { "tp": 17682, "tn": 0, "fp": 4106, "fn": 132 } }, "thr0.70": { "need_precision": 0.8112270700197692, "need_recall": 0.9905130796003143, "need_f1": 0.8919499557689878, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8049726277372263, "known_question_refusal_rate": 0.00948692039968564, "unknown_question_read_rate": 1.0, "counts": { "tp": 17645, "tn": 0, "fp": 4106, "fn": 169 } }, "thr0.80": { "need_precision": 0.8106525247867189, "need_recall": 0.9868081284383069, "need_f1": 0.8900984835059115, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8019616788321168, "known_question_refusal_rate": 0.01319187156169305, "unknown_question_read_rate": 1.0, "counts": { "tp": 17579, "tn": 0, "fp": 4106, "fn": 235 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_2": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.9999341862276196, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 1.0, "counts": { "tp": 0, "tn": 0, "fp": 64, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.442375, "recall1": 0.39675, "recall3": 0.49153125, "recall5": 0.50303125, "recall8": 0.51234375, "evidence_count_in_top3": 0.57475, "hit3": 0.5283125, "mrr": 0.500402578720693, "ndcg3": 0.4692792057097287, "mean_first_positive_rank": 14.9525, "all_evidence_in_top3": 0.45475, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.3715, "hop_accuracy": 0.74955, "hop_accuracy_answerable": 0.9369375, "hop_under_prediction": 0.0160625, "hop_over_prediction": 0.047, "mean_score_margin": -8.091850390457548, "median_score_margin": -6.662378132343292, "negative_margin_rate": 0.557625, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 4000, "true_0_pred_2": 0, "true_1_pred_0": 257, "true_1_pred_1": 12991, "true_1_pred_2": 752, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9821553228273988, "unknown_mean_need_prob": 0.9988265322595835, "thr0.50": { "need_precision": 0.8, "need_recall": 1.0, "need_f1": 0.888888888888889, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 16000, "tn": 0, "fp": 4000, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.14528412419449327, "recall1": 0.14528412419449327, "recall3": 0.2677211482132396, "recall5": 0.38488576449912126, "recall8": 0.5653192735793791, "evidence_count_in_top3": 0.2677211482132396, "hit3": 0.2677211482132396, "mrr": 0.2801716446629072, "ndcg3": 0.21516992510461316, "mean_first_positive_rank": 9.195664909197422, "all_evidence_in_top3": 0.2677211482132396, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.5740740740740741, "hop_accuracy_answerable": 0.5811364967779731, "hop_under_prediction": 0.3380199179847686, "hop_over_prediction": 0.08084358523725835, "mean_score_margin": -19.359866383266713, "median_score_margin": -18.159542322158813, "negative_margin_rate": 0.8547158758055068, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 21, "true_0_pred_2": 0, "true_1_pred_0": 577, "true_1_pred_1": 992, "true_1_pred_2": 138 }, "known_mean_need_prob": 0.9195722371566764, "unknown_mean_need_prob": 0.9999671209426153, "thr0.50": { "need_precision": 0.9876179245283019, "need_recall": 0.981253661394259, "need_f1": 0.9844255069056715, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.9693287037037037, "known_question_refusal_rate": 0.018746338605741066, "unknown_question_read_rate": 1.0, "counts": { "tp": 1675, "tn": 0, "fp": 21, "fn": 32 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.0, "recall1": 0.0, "recall3": 0.028037383177570093, "recall5": 0.07476635514018691, "recall8": 0.2803738317757009, "evidence_count_in_top3": 0.028037383177570093, "hit3": 0.028037383177570093, "mrr": 0.1023068389893556, "ndcg3": 0.01524233414552764, "mean_first_positive_rank": 12.598130841121495, "all_evidence_in_top3": 0.028037383177570093, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.5390625, "hop_accuracy_answerable": 0.6448598130841121, "hop_under_prediction": 0.0, "hop_over_prediction": 0.35514018691588783, "mean_score_margin": -29.291684916086286, "median_score_margin": -29.07153034210205, "negative_margin_rate": 1.0, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 21, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 69, "true_1_pred_2": 38 }, "known_mean_need_prob": 0.9999633771236812, "unknown_mean_need_prob": 0.9999671209426153, "thr0.50": { "need_precision": 0.8359375, "need_recall": 1.0, "need_f1": 0.9106382978723404, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8359375, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 1.0, "counts": { "tp": 107, "tn": 0, "fp": 21, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.796, "recall1": 0.796, "recall3": 0.9125, "recall5": 0.919, "recall8": 0.927, "evidence_count_in_top3": 0.9125, "hit3": 0.9125, "mrr": 0.8582544951434297, "ndcg3": 0.8678012294946459, "mean_first_positive_rank": 2.8275, "all_evidence_in_top3": 0.9125, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.4955, "hop_accuracy_answerable": 0.4955, "hop_under_prediction": 0.1285, "hop_over_prediction": 0.376, "mean_score_margin": 21.036133953779935, "median_score_margin": 21.400100469589233, "negative_margin_rate": 0.204, "hop_confusion": { "true_1_pred_0": 257, "true_1_pred_1": 991, "true_1_pred_2": 752 }, "known_mean_need_prob": 0.9764632563889026, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.231, "recall1": 0.231, "recall3": 0.2535, "recall5": 0.255, "recall8": 0.257, "evidence_count_in_top3": 0.2535, "hit3": 0.2535, "mrr": 0.26603145346142226, "ndcg3": 0.24486859507142916, "mean_first_positive_rank": 23.789, "all_evidence_in_top3": 0.2535, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -27.718347060799598, "median_score_margin": -38.59446668624878, "negative_margin_rate": 0.769, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9723067943453789, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.2255, "recall1": 0.2255, "recall3": 0.2455, "recall5": 0.2475, "recall8": 0.2485, "evidence_count_in_top3": 0.2455, "hit3": 0.2455, "mrr": 0.2596338049221357, "ndcg3": 0.23792220044107196, "mean_first_positive_rank": 24.0695, "all_evidence_in_top3": 0.2455, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -28.903205164782705, "median_score_margin": -39.01692509651184, "negative_margin_rate": 0.7745, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9724746745526791, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.99774328109622, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 1.0, "counts": { "tp": 0, "tn": 0, "fp": 2000, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.5255, "recall1": 0.5255, "recall3": 0.6245, "recall5": 0.6265, "recall8": 0.6295, "evidence_count_in_top3": 0.6245, "hit3": 0.6245, "mrr": 0.5855216777342396, "ndcg3": 0.585867169546431, "mean_first_positive_rank": 12.2085, "all_evidence_in_top3": 0.6245, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -1.0039909677058458, "median_score_margin": 2.929476022720337, "negative_margin_rate": 0.4745, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9875293919146061, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.73, "recall1": 0.365, "recall3": 0.66575, "recall5": 0.72725, "recall8": 0.77375, "evidence_count_in_top3": 1.3315, "hit3": 0.96, "mrr": 0.84532448592014, "ndcg3": 0.6658792959224611, "mean_first_positive_rank": 1.59, "all_evidence_in_top3": 0.3715, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.3715, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.786984693795443, "median_score_margin": 5.27826452255249, "negative_margin_rate": 0.27, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9993210740983486, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.443, "recall1": 0.443, "recall3": 0.5525, "recall5": 0.5615, "recall8": 0.5665, "evidence_count_in_top3": 0.5525, "hit3": 0.5525, "mrr": 0.5118924660255056, "ndcg3": 0.5092063534375025, "mean_first_positive_rank": 13.8255, "all_evidence_in_top3": 0.5525, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -2.995655853047967, "median_score_margin": -5.5251545906066895, "negative_margin_rate": 0.557, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9880377069413662, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.249, "recall1": 0.249, "recall3": 0.287, "recall5": 0.293, "recall8": 0.3005, "evidence_count_in_top3": 0.287, "hit3": 0.287, "mrr": 0.2919631936667751, "ndcg3": 0.27173149797678653, "mean_first_positive_rank": 22.067, "all_evidence_in_top3": 0.287, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -19.72415616109967, "median_score_margin": -21.928287506103516, "negative_margin_rate": 0.751, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9656237106621265, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.339, "recall1": 0.339, "recall3": 0.391, "recall5": 0.3945, "recall8": 0.396, "evidence_count_in_top3": 0.391, "hit3": 0.391, "mrr": 0.3845990528918967, "ndcg3": 0.3709573037875013, "mean_first_positive_rank": 19.243, "all_evidence_in_top3": 0.391, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -8.212566563799978, "median_score_margin": -16.937832549214363, "negative_margin_rate": 0.661, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9954859737157822, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.13671444321940462, "recall1": 0.13671444321940462, "recall3": 0.2535832414553473, "recall5": 0.36659316427783906, "recall8": 0.5485115766262404, "evidence_count_in_top3": 0.2535832414553473, "hit3": 0.2535832414553473, "mrr": 0.26968017045834825, "ndcg3": 0.2033770627933551, "mean_first_positive_rank": 9.396361631753033, "all_evidence_in_top3": 0.2535832414553473, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.5526041666666667, "hop_accuracy_answerable": 0.5848952590959207, "hop_under_prediction": 0.31808158765159866, "hop_over_prediction": 0.0970231532524807, "mean_score_margin": -19.945701324287494, "median_score_margin": -18.862584590911865, "negative_margin_rate": 0.8632855567805954, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 42, "true_0_pred_2": 64, "true_1_pred_0": 577, "true_1_pred_1": 1061, "true_1_pred_2": 176 }, "known_mean_need_prob": 0.9243141621712682, "unknown_mean_need_prob": 0.9999472358316746, "thr0.50": { "need_precision": 0.9438559322033898, "need_recall": 0.9823594266813671, "need_f1": 0.9627228525121555, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.928125, "known_question_refusal_rate": 0.017640573318632856, "unknown_question_read_rate": 1.0, "counts": { "tp": 1782, "tn": 0, "fp": 106, "fn": 32 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 0, "true_0_pred_1": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.9999097834229469, "thr0.50": { "need_precision": 0.0, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 1.0, "counts": { "tp": 0, "tn": 0, "fp": 2000, "fn": 0 } } } } }, "REPLAY-128 v7": { "checkpoint": "checkpoints\\router_replay_v7_v2_128\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 8156921 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 8156921 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.9359393330911795, "single_query_latency_ms_p50": 0.8711999980732799, "single_query_latency_ms_p95": 1.5259000065270811, "queries_per_second": 1068.445319738026, "samples": 150, "batched_qps_64": 65927.72676538704, "batched_latency_ms_64": 0.015168124991760124, "batched_qps_256": 262171.5169506191, "batched_latency_ms_256": 0.003814296883319912 }, "cpu": { "single_query_latency_ms_mean": 0.40274533355841413, "single_query_latency_ms_p50": 0.36700000055134296, "single_query_latency_ms_p95": 0.6454000103985891, "queries_per_second": 2482.9586259004045, "samples": 150, "batched_qps_64": 12091.395836736374, "batched_latency_ms_64": 0.08270343751064502, "batched_qps_256": 12591.79467573841, "batched_latency_ms_256": 0.07941679687064607 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9413944088918828, "recall1": 0.8857920736499383, "recall3": 0.9647748961490962, "recall5": 0.9714550353654429, "recall8": 0.9804086673402942, "evidence_count_in_top3": 1.0744919726058157, "hit3": 0.9673290670259347, "mrr": 0.957689277532011, "ndcg3": 0.9536787031890833, "mean_first_positive_rank": 1.34001347254968, "all_evidence_in_top3": 0.9622207252722578, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9545, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 40.14557690149532, "median_score_margin": 45.34479379653931, "negative_margin_rate": 0.05860559110811721, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999984274064, "unknown_mean_need_prob": 4.836221294362452e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 4.8406160974828055e-06, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.98225, "recall1": 0.92034375, "recall3": 0.99715625, "recall5": 0.99959375, "recall8": 0.9996875, "evidence_count_in_top3": 1.1193125, "hit3": 1.0, "mrr": 0.9911041666666667, "ndcg3": 0.9893498205729669, "mean_first_positive_rank": 1.017875, "all_evidence_in_top3": 0.9943125, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9545, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 44.73340179644042, "median_score_margin": 47.92497134208679, "negative_margin_rate": 0.01775, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999701977, "unknown_mean_need_prob": 6.379481702447893e-11, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.5875805506736965, "recall1": 0.5875805506736965, "recall3": 0.6731107205623902, "recall5": 0.7106033977738723, "recall8": 0.7996485061511424, "evidence_count_in_top3": 0.6731107205623902, "hit3": 0.6731107205623902, "mrr": 0.6633131807141506, "ndcg3": 0.6380925044585338, "mean_first_positive_rank": 4.295254833040421, "all_evidence_in_top3": 0.6731107205623902, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.2721888081208264, "median_score_margin": 8.391286849975586, "negative_margin_rate": 0.4124194493263035, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999838679872, "unknown_mean_need_prob": 0.0004654159529796483, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.4766355140186916, "recall1": 0.4766355140186916, "recall3": 0.7757009345794392, "recall5": 0.9252336448598131, "recall8": 0.9813084112149533, "evidence_count_in_top3": 0.7757009345794392, "hit3": 0.7757009345794392, "mrr": 0.6573319982198488, "ndcg3": 0.6543120031041451, "mean_first_positive_rank": 2.364485981308411, "all_evidence_in_top3": 0.7757009345794392, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -1.0887432181946586, "median_score_margin": -0.6999244689941406, "negative_margin_rate": 0.5233644859813084, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0004654159529796483, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.991, "recall1": 0.991, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9955, "ndcg3": 0.9966783677821431, "mean_first_positive_rank": 1.009, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 42.24227243242413, "median_score_margin": 42.77137851715088, "negative_margin_rate": 0.009, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.973, "recall1": 0.973, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9865, "ndcg3": 0.9900351033464293, "mean_first_positive_rank": 1.027, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 50.82861385926604, "median_score_margin": 54.98523235321045, "negative_margin_rate": 0.027, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.983, "recall1": 0.983, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9915, "ndcg3": 0.9937258058107148, "mean_first_positive_rank": 1.017, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 51.79758185644448, "median_score_margin": 55.65491724014282, "negative_margin_rate": 0.017, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.275862421441728e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.978, "recall1": 0.978, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9889166666666667, "ndcg3": 0.9918149897017863, "mean_first_positive_rank": 1.0225, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 47.034401270009575, "median_score_margin": 50.28937745094299, "negative_margin_rate": 0.022, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9905, "recall1": 0.49525, "recall3": 0.97725, "recall5": 0.99675, "recall8": 0.9975, "evidence_count_in_top3": 1.9545, "hit3": 1.0, "mrr": 0.99525, "ndcg3": 0.963831301989089, "mean_first_positive_rank": 1.0095, "all_evidence_in_top3": 0.9545, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9545, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 46.103771095309405, "median_score_margin": 52.25391101837158, "negative_margin_rate": 0.0095, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999997615814, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.994, "recall1": 0.994, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.997, "ndcg3": 0.9977855785214288, "mean_first_positive_rank": 1.006, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 36.59333083114028, "median_score_margin": 35.444119691848755, "negative_margin_rate": 0.006, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.978, "recall1": 0.978, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9889166666666667, "ndcg3": 0.9918149897017863, "mean_first_positive_rank": 1.0225, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 35.780669214252384, "median_score_margin": 35.064393758773804, "negative_margin_rate": 0.022, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9705, "recall1": 0.9705, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98525, "ndcg3": 0.989112427730358, "mean_first_positive_rank": 1.0295, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 47.48657381267706, "median_score_margin": 51.48151856660843, "negative_margin_rate": 0.0295, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.5810363836824697, "recall1": 0.5810363836824697, "recall3": 0.679162072767365, "recall5": 0.7232635060639471, "recall8": 0.8103638368246968, "evidence_count_in_top3": 0.679162072767365, "hit3": 0.679162072767365, "mrr": 0.662960376675071, "ndcg3": 0.6390492224051051, "mean_first_positive_rank": 4.181367144432194, "all_evidence_in_top3": 0.679162072767365, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.32035381466873164, "median_score_margin": 7.441610336303711, "negative_margin_rate": 0.4189636163175303, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999848195447, "unknown_mean_need_prob": 0.00018733273071117103, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 3.3919047850488803e-15, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "PROD-128 step20000": { "checkpoint": "checkpoints\\router_prod_v2_128\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 8156921 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 8156921 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.9775833330544023, "single_query_latency_ms_p50": 0.9395999950356781, "single_query_latency_ms_p95": 1.3619999954244122, "queries_per_second": 1022.9306967371857, "samples": 150, "batched_qps_64": 73335.62500742644, "batched_latency_ms_64": 0.013635937511935481, "batched_qps_256": 273837.7939855633, "batched_latency_ms_256": 0.0036517968737825868 }, "cpu": { "single_query_latency_ms_mean": 0.39715666566432145, "single_query_latency_ms_p50": 0.3654000029200688, "single_query_latency_ms_p95": 0.6416000105673447, "queries_per_second": 2517.8980650552758, "samples": 150, "batched_qps_64": 11271.852422702024, "batched_latency_ms_64": 0.08871656250448723, "batched_qps_256": 13151.297458363848, "batched_latency_ms_256": 0.07603812499610285 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9378017289772089, "recall1": 0.8834343774559336, "recall3": 0.9658976086224318, "recall5": 0.9732794431346132, "recall8": 0.9842258897496351, "evidence_count_in_top3": 1.0772426181654877, "hit3": 0.9668238464129336, "mrr": 0.9559179074946493, "ndcg3": 0.9538607332074894, "mean_first_positive_rank": 1.2769731671718874, "all_evidence_in_top3": 0.96497137083193, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9835, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 21.892463866317613, "median_score_margin": 25.104029178619385, "negative_margin_rate": 0.06219827102279106, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999600001406399, "unknown_mean_need_prob": 0.0011980682588774603, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.010227199368273432, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.9809375, "recall1": 0.92040625, "recall3": 0.99896875, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.1229375, "hit3": 1.0, "mrr": 0.9904270833333334, "ndcg3": 0.99126750245029, "mean_first_positive_rank": 1.0193125, "all_evidence_in_top3": 0.9979375, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9835, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 24.198241920885398, "median_score_margin": 26.095373153686523, "negative_margin_rate": 0.0190625, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999700157120824, "unknown_mean_need_prob": 6.808238428402546e-05, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.5764499121265377, "recall1": 0.5764499121265377, "recall3": 0.6795547744581136, "recall5": 0.7352079671939075, "recall8": 0.8412419449326304, "evidence_count_in_top3": 0.6795547744581136, "hit3": 0.6795547744581136, "mrr": 0.6631831932582257, "ndcg3": 0.635135598759312, "mean_first_positive_rank": 3.5219683655536027, "all_evidence_in_top3": 0.6795547744581136, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 1.810913933616723, "median_score_margin": 3.2161903381347656, "negative_margin_rate": 0.4235500878734622, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9998666250922209, "unknown_mean_need_prob": 0.09505709462488691, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.2523364485981308, "recall1": 0.2523364485981308, "recall3": 0.5887850467289719, "recall5": 0.7757009345794392, "recall8": 0.9065420560747663, "evidence_count_in_top3": 0.5887850467289719, "hit3": 0.5887850467289719, "mrr": 0.4657435503229896, "ndcg3": 0.44503359879840326, "mean_first_positive_rank": 3.9906542056074765, "all_evidence_in_top3": 0.5887850467289719, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -2.5303691987679384, "median_score_margin": -1.9369163513183594, "negative_margin_rate": 0.7476635514018691, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 0.9999519820525268, "unknown_mean_need_prob": 0.09505709462488691, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.993, "recall1": 0.993, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9965, "ndcg3": 0.9974165082750002, "mean_first_positive_rank": 1.007, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 26.333668621126563, "median_score_margin": 27.148656368255615, "negative_margin_rate": 0.007, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.999947308152914, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9755, "recall1": 0.9755, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98775, "ndcg3": 0.9909577789625007, "mean_first_positive_rank": 1.0245, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 25.041911346442998, "median_score_margin": 26.2372727394104, "negative_margin_rate": 0.0245, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.999997846186161, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.983, "recall1": 0.983, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9915, "ndcg3": 0.9937258058107148, "mean_first_positive_rank": 1.017, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 25.644431973803787, "median_score_margin": 26.433398723602295, "negative_margin_rate": 0.017, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999978821873665, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.00013525628542129198, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9845, "recall1": 0.9845, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9921666666666666, "ndcg3": 0.9942139463035718, "mean_first_positive_rank": 1.016, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 23.74752949652076, "median_score_margin": 26.17783832550049, "negative_margin_rate": 0.0155, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999990122318267, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9685, "recall1": 0.48425, "recall3": 0.99175, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.9835, "hit3": 1.0, "mrr": 0.9840833333333334, "ndcg3": 0.9749284491737452, "mean_first_positive_rank": 1.0325, "all_evidence_in_top3": 0.9835, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9835, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 20.615374811934306, "median_score_margin": 23.414374619722366, "negative_margin_rate": 0.0315, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999314472973346, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9955, "recall1": 0.9955, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99775, "ndcg3": 0.9983391838910716, "mean_first_positive_rank": 1.0045, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 22.83826958390698, "median_score_margin": 24.362728118896484, "negative_margin_rate": 0.0045, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999094858169556, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9815, "recall1": 0.9815, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99075, "ndcg3": 0.9931722004410719, "mean_first_positive_rank": 1.0185, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 25.664584860108793, "median_score_margin": 26.572662353515625, "negative_margin_rate": 0.0185, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999773879647255, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.966, "recall1": 0.966, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9829166666666667, "ndcg3": 0.9873861467446439, "mean_first_positive_rank": 1.0345, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 23.700164673238994, "median_score_margin": 25.87112593650818, "negative_margin_rate": 0.034, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.999999755859375, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.5573318632855567, "recall1": 0.5573318632855567, "recall3": 0.6742006615214995, "recall5": 0.7375964718853363, "recall8": 0.8450937155457552, "evidence_count_in_top3": 0.6742006615214995, "hit3": 0.6742006615214995, "mrr": 0.6515370842207008, "ndcg3": 0.6239223054870864, "mean_first_positive_rank": 3.549614112458655, "all_evidence_in_top3": 0.6742006615214995, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 1.5548404522687853, "median_score_margin": 2.31545352935791, "negative_margin_rate": 0.4426681367144432, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9998716599294606, "unknown_mean_need_prob": 0.043839044658629714, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 9.084831467589538e-07, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "PROD-128 step10000": { "checkpoint": "checkpoints\\router_prod_v2_128\\router_step_00010000.pt", "router": { "kind": "router_v2", "step": 10000, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 24473667 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 24473667 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.2315193335719716, "single_query_latency_ms_p50": 0.9520000021439046, "single_query_latency_ms_p95": 1.6566999984206632, "queries_per_second": 812.0051165575621, "samples": 150, "batched_qps_64": 49115.91353738528, "batched_latency_ms_64": 0.020360000007713097, "batched_qps_256": 141564.73262936738, "batched_latency_ms_256": 0.007063906252824381 }, "cpu": { "single_query_latency_ms_mean": 0.46193000006799895, "single_query_latency_ms_p50": 0.3951999970013276, "single_query_latency_ms_p95": 0.8545000018784776, "queries_per_second": 2164.83016875456, "samples": 150, "batched_qps_64": 10985.200876163763, "batched_latency_ms_64": 0.09103156248784217, "batched_qps_256": 12158.516660730622, "batched_latency_ms_256": 0.08224687500160144 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9380824070955428, "recall1": 0.8837711911979342, "recall3": 0.9677500842034354, "recall5": 0.9777702930279556, "recall8": 0.9879869765353093, "evidence_count_in_top3": 1.0793196362411586, "hit3": 0.9684517794992702, "mrr": 0.9570269878646712, "ndcg3": 0.9552694474506009, "mean_first_positive_rank": 1.2445829123161558, "all_evidence_in_top3": 0.9670483889076008, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9875, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 18.42186796360512, "median_score_margin": 18.455811977386475, "negative_margin_rate": 0.06191759290445717, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999600001406399, "unknown_mean_need_prob": 0.0011980682588774603, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.010227199368273432, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.9798125, "recall1": 0.91934375, "recall3": 0.99921875, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.1234375, "hit3": 1.0, "mrr": 0.989875, "ndcg3": 0.991227295889911, "mean_first_positive_rank": 1.020375, "all_evidence_in_top3": 0.9984375, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9875, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 20.280915253964952, "median_score_margin": 19.570133835077286, "negative_margin_rate": 0.0201875, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999700157120824, "unknown_mean_need_prob": 6.808238428402546e-05, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.5922671353251318, "recall1": 0.5922671353251318, "recall3": 0.700058582308143, "recall5": 0.7844171060339777, "recall8": 0.8816637375512595, "evidence_count_in_top3": 0.700058582308143, "hit3": 0.700058582308143, "mrr": 0.6824865585659331, "ndcg3": 0.6535262192986877, "mean_first_positive_rank": 3.1646162858816638, "all_evidence_in_top3": 0.700058582308143, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.295050391936091, "median_score_margin": 2.6275386810302734, "negative_margin_rate": 0.4077328646748682, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9998666250922209, "unknown_mean_need_prob": 0.09505709462488691, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.21495327102803738, "recall1": 0.21495327102803738, "recall3": 0.5327102803738317, "recall5": 0.7383177570093458, "recall8": 0.8878504672897196, "evidence_count_in_top3": 0.5327102803738317, "hit3": 0.5327102803738317, "mrr": 0.42499258270286305, "ndcg3": 0.3921864140520735, "mean_first_positive_rank": 4.140186915887851, "all_evidence_in_top3": 0.5327102803738317, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -2.2910203627336805, "median_score_margin": -1.6779632568359375, "negative_margin_rate": 0.7850467289719626, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 0.9999519820525268, "unknown_mean_need_prob": 0.09505709462488691, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9925, "recall1": 0.9925, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99625, "ndcg3": 0.9972319731517859, "mean_first_positive_rank": 1.0075, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 20.927725780989974, "median_score_margin": 18.922175407409668, "negative_margin_rate": 0.0075, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.999947308152914, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.974, "recall1": 0.974, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.987, "ndcg3": 0.9904041735928579, "mean_first_positive_rank": 1.026, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 21.25606323112175, "median_score_margin": 19.60146927833557, "negative_margin_rate": 0.026, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.999997846186161, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9815, "recall1": 0.9815, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99075, "ndcg3": 0.9931722004410719, "mean_first_positive_rank": 1.0185, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 21.907086151573807, "median_score_margin": 19.700664520263672, "negative_margin_rate": 0.0185, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999978821873665, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.00013525628542129198, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.979, "recall1": 0.979, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9894166666666666, "ndcg3": 0.9921840599482149, "mean_first_positive_rank": 1.0215, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 19.10362393591553, "median_score_margin": 18.17929482460022, "negative_margin_rate": 0.021, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999990122318267, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9675, "recall1": 0.48375, "recall3": 0.99375, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.9875, "hit3": 1.0, "mrr": 0.9835833333333334, "ndcg3": 0.9774938937853556, "mean_first_positive_rank": 1.0335, "all_evidence_in_top3": 0.9875, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9875, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 18.34438194381073, "median_score_margin": 20.793008476495743, "negative_margin_rate": 0.0325, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999314472973346, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9925, "recall1": 0.9925, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99625, "ndcg3": 0.9972319731517859, "mean_first_positive_rank": 1.0075, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 18.88056689170003, "median_score_margin": 17.6385018825531, "negative_margin_rate": 0.0075, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999094858169556, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.979, "recall1": 0.979, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9895, "ndcg3": 0.9922495248250006, "mean_first_positive_rank": 1.021, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 22.230981560852378, "median_score_margin": 20.683798789978027, "negative_margin_rate": 0.021, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999773879647255, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9725, "recall1": 0.9725, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98625, "ndcg3": 0.989850568223215, "mean_first_positive_rank": 1.0275, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 19.596892535755412, "median_score_margin": 18.929328441619873, "negative_margin_rate": 0.0275, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.999999755859375, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.5700110253583242, "recall1": 0.5700110253583242, "recall3": 0.6901874310915105, "recall5": 0.7816979051819184, "recall8": 0.8820286659316428, "evidence_count_in_top3": 0.6901874310915105, "hit3": 0.6901874310915105, "mrr": 0.6672981046423672, "ndcg3": 0.6381109165636338, "mean_first_positive_rank": 3.2221609702315326, "all_evidence_in_top3": 0.6901874310915105, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.0245379494059557, "median_score_margin": 1.7993928790092468, "negative_margin_rate": 0.42998897464167585, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9998716599294606, "unknown_mean_need_prob": 0.043839044658629714, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 9.084831467589538e-07, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "V2-128 v6 best": { "checkpoint": "checkpoints\\router_v6_v2_128\\router_best.pt", "router": { "kind": "router_v2", "step": 6000, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 24472973 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 24472973 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.2807459999263908, "single_query_latency_ms_p50": 1.0074999881908298, "single_query_latency_ms_p95": 1.8933000101242214, "queries_per_second": 780.7949429921887, "samples": 150, "batched_qps_64": 62429.27930602223, "batched_latency_ms_64": 0.016018125006667105, "batched_qps_256": 201612.9031802962, "batched_latency_ms_256": 0.004960000001119624 }, "cpu": { "single_query_latency_ms_mean": 0.4724313324550167, "single_query_latency_ms_p50": 0.40469999657943845, "single_query_latency_ms_p95": 0.8424999978160486, "queries_per_second": 2116.7097338854355, "samples": 150, "batched_qps_64": 10979.62250423899, "batched_latency_ms_64": 0.09107781252168934, "batched_qps_256": 12091.087451043528, "batched_latency_ms_256": 0.08270554687896947 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.946165936903559, "recall1": 0.8919108566296171, "recall3": 0.9780509711462895, "recall5": 0.984001347254968, "recall8": 0.9933198607836533, "evidence_count_in_top3": 1.0897608622431796, "hit3": 0.9786123273829572, "mrr": 0.9648813461092199, "ndcg3": 0.9649494672503449, "mean_first_positive_rank": 1.1781744695183562, "all_evidence_in_top3": 0.9774896149096216, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.99, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 14.959203617273067, "median_score_margin": 14.255205273628235, "negative_margin_rate": 0.053834063096441, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999601252354449, "unknown_mean_need_prob": 0.0012111918483046415, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.010367806648901023, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.976875, "recall1": 0.91646875, "recall3": 0.999375, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.12375, "hit3": 1.0, "mrr": 0.9883645833333333, "ndcg3": 0.9901670270848972, "mean_first_positive_rank": 1.0235625, "all_evidence_in_top3": 0.99875, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.99, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 16.402275828796906, "median_score_margin": 15.20901894569397, "negative_margin_rate": 0.023125, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999701014049351, "unknown_mean_need_prob": 6.905754663642937e-05, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.6854130052724078, "recall1": 0.6854130052724078, "recall3": 0.786174575278266, "recall5": 0.836555360281195, "recall8": 0.9302870533099004, "evidence_count_in_top3": 0.786174575278266, "hit3": 0.786174575278266, "mrr": 0.7618258791916341, "ndcg3": 0.7445377808477717, "mean_first_positive_rank": 2.570591681312244, "all_evidence_in_top3": 0.786174575278266, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.378015720335124, "median_score_margin": 3.452073097229004, "negative_margin_rate": 0.3145869947275923, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9998671145696143, "unknown_mean_need_prob": 0.0960329504062732, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.514018691588785, "recall1": 0.514018691588785, "recall3": 0.8504672897196262, "recall5": 0.9439252336448598, "recall8": 1.0, "evidence_count_in_top3": 0.8504672897196262, "hit3": 0.8504672897196262, "mrr": 0.6927681352914997, "ndcg3": 0.7103867694592852, "mean_first_positive_rank": 2.0841121495327104, "all_evidence_in_top3": 0.8504672897196262, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.11619492766456069, "median_score_margin": 0.07718276977539062, "negative_margin_rate": 0.48598130841121495, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 0.9999521859338351, "unknown_mean_need_prob": 0.0960329504062732, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9885, "recall1": 0.9885, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99425, "ndcg3": 0.9957556921660717, "mean_first_positive_rank": 1.0115, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 18.511014288797973, "median_score_margin": 17.0071280002594, "negative_margin_rate": 0.0115, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999475074112415, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9755, "recall1": 0.9755, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98775, "ndcg3": 0.9909577789625007, "mean_first_positive_rank": 1.0245, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 17.432058019019664, "median_score_margin": 17.119813442230225, "negative_margin_rate": 0.0245, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.999997846186161, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9805, "recall1": 0.9805, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99025, "ndcg3": 0.9928031301946435, "mean_first_positive_rank": 1.0195, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 18.200293550987727, "median_score_margin": 18.367231130599976, "negative_margin_rate": 0.0195, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999978821873665, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.00013719209501925888, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.98, "recall1": 0.98, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9899166666666667, "ndcg3": 0.9925531301946434, "mean_first_positive_rank": 1.0205, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 17.648428340390325, "median_score_margin": 17.865666389465332, "negative_margin_rate": 0.02, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999990122318267, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9665, "recall1": 0.48325, "recall3": 0.995, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.99, "hit3": 1.0, "mrr": 0.983, "ndcg3": 0.9775122185202447, "mean_first_positive_rank": 1.035, "all_evidence_in_top3": 0.99, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.99, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 10.340590946737677, "median_score_margin": 12.604959726333618, "negative_margin_rate": 0.0335, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999316669106484, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9855, "recall1": 0.9855, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99275, "ndcg3": 0.9946484814267861, "mean_first_positive_rank": 1.0145, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 16.270763264942914, "median_score_margin": 14.272640228271484, "negative_margin_rate": 0.0145, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.999909707903862, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.978, "recall1": 0.978, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.989, "ndcg3": 0.9918804545785721, "mean_first_positive_rank": 1.022, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 17.229778190023264, "median_score_margin": 17.07593348622322, "negative_margin_rate": 0.022, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999774325489997, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9605, "recall1": 0.9605, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98, "ndcg3": 0.9852253306357154, "mean_first_positive_rank": 1.041, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 15.585280029475689, "median_score_margin": 15.148456335067749, "negative_margin_rate": 0.0395, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.999999755859375, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.675303197353914, "recall1": 0.675303197353914, "recall3": 0.7899669239250275, "recall5": 0.8428886438809261, "recall8": 0.934399117971334, "evidence_count_in_top3": 0.7899669239250275, "hit3": 0.7899669239250275, "mrr": 0.7577524621038092, "ndcg3": 0.74252336066113, "mean_first_positive_rank": 2.541896361631753, "all_evidence_in_top3": 0.7899669239250275, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.2308930415391117, "median_score_margin": 3.026643395423889, "negative_margin_rate": 0.324696802646086, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9998721325607783, "unknown_mean_need_prob": 0.04431059945842585, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 9.229982535998715e-07, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } } }