{ "V2-128 v6 final": { "checkpoint": "checkpoints\\router_v6_v2_128\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 8156921 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 8156921 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.8988993330664622, "single_query_latency_ms_p50": 0.869700001203455, "single_query_latency_ms_p95": 1.2395000012475066, "queries_per_second": 1112.4716230333022, "samples": 150, "batched_qps_64": 67531.91938657848, "batched_latency_ms_64": 0.014807812499384454, "batched_qps_256": 237234.73279415478, "batched_latency_ms_256": 0.004215234372395571 }, "cpu": { "single_query_latency_ms_mean": 0.36343133309856057, "single_query_latency_ms_p50": 0.3530000030878, "single_query_latency_ms_p95": 0.5296000017551705, "queries_per_second": 2751.5514181844237, "samples": 150, "batched_qps_64": 14346.173160954218, "batched_latency_ms_64": 0.0697049999871524, "batched_qps_256": 13080.658610801669, "batched_latency_ms_256": 0.0764487500021005 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9436959694622207, "recall1": 0.8882059054676097, "recall3": 0.9639609296059279, "recall5": 0.973981138430448, "recall8": 0.9864151790726395, "evidence_count_in_top3": 1.0717974626698104, "hit3": 0.9683956438756035, "mrr": 0.9597054333626427, "ndcg3": 0.9530820526370456, "mean_first_positive_rank": 1.2573818345121814, "all_evidence_in_top3": 0.9595262153362524, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.921, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 51.315328869764656, "median_score_margin": 56.234654903411865, "negative_margin_rate": 0.05630403053777928, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999984274064, "unknown_mean_need_prob": 4.836221294362452e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 4.8406160974828055e-06, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.9838125, "recall1": 0.92203125, "recall3": 0.9950625, "recall5": 0.99978125, "recall8": 0.999875, "evidence_count_in_top3": 1.115125, "hit3": 1.0, "mrr": 0.991875, "ndcg3": 0.9876817889701721, "mean_first_positive_rank": 1.016375, "all_evidence_in_top3": 0.990125, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.921, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 57.110110210259094, "median_score_margin": 59.90444374084473, "negative_margin_rate": 0.0161875, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999701977, "unknown_mean_need_prob": 6.379481702447893e-11, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.5940246045694201, "recall1": 0.5940246045694201, "recall3": 0.6842413591095489, "recall5": 0.7352079671939075, "recall8": 0.8611599297012302, "evidence_count_in_top3": 0.6842413591095489, "hit3": 0.6842413591095489, "mrr": 0.6760273585431921, "ndcg3": 0.6466497456649108, "mean_first_positive_rank": 3.4463971880492092, "all_evidence_in_top3": 0.6842413591095489, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.27154229221702947, "median_score_margin": 7.975786209106445, "negative_margin_rate": 0.40597539543058, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999838679872, "unknown_mean_need_prob": 0.0004654159529796483, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.5233644859813084, "recall1": 0.5233644859813084, "recall3": 0.7757009345794392, "recall5": 0.9252336448598131, "recall8": 0.9719626168224299, "evidence_count_in_top3": 0.7757009345794392, "hit3": 0.7757009345794392, "mrr": 0.674896157840083, "ndcg3": 0.6678873486315127, "mean_first_positive_rank": 2.3738317757009346, "all_evidence_in_top3": 0.7757009345794392, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.8786688875929217, "median_score_margin": 0.4756507873535156, "negative_margin_rate": 0.4766355140186916, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0004654159529796483, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.993, "recall1": 0.993, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9965, "ndcg3": 0.9974165082750002, "mean_first_positive_rank": 1.007, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 54.4210530708991, "median_score_margin": 56.92102670669556, "negative_margin_rate": 0.007, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.975, "recall1": 0.975, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9875, "ndcg3": 0.9907732438392864, "mean_first_positive_rank": 1.025, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 64.29146421498805, "median_score_margin": 73.40720510482788, "negative_margin_rate": 0.025, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9845, "recall1": 0.9845, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99225, "ndcg3": 0.9942794111803576, "mean_first_positive_rank": 1.0155, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 65.52974343589041, "median_score_margin": 73.97303533554077, "negative_margin_rate": 0.0155, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.275862421441728e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9795, "recall1": 0.9795, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9896666666666666, "ndcg3": 0.9923685950714292, "mean_first_positive_rank": 1.021, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 60.150181606799364, "median_score_margin": 68.01980018615723, "negative_margin_rate": 0.0205, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9885, "recall1": 0.49425, "recall3": 0.9605, "recall5": 0.99825, "recall8": 0.999, "evidence_count_in_top3": 1.921, "hit3": 1.0, "mrr": 0.99425, "ndcg3": 0.9452009954703022, "mean_first_positive_rank": 1.0115, "all_evidence_in_top3": 0.921, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.921, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 55.6439196386747, "median_score_margin": 62.50904202461243, "negative_margin_rate": 0.0115, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999997615814, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.995, "recall1": 0.995, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9975, "ndcg3": 0.9981546487678573, "mean_first_positive_rank": 1.005, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 48.13904330630228, "median_score_margin": 49.25835180282593, "negative_margin_rate": 0.005, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9805, "recall1": 0.9805, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9901666666666666, "ndcg3": 0.9927376653178577, "mean_first_positive_rank": 1.02, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 50.85701001610607, "median_score_margin": 54.9089093208313, "negative_margin_rate": 0.0195, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9745, "recall1": 0.9745, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9871666666666666, "ndcg3": 0.9905232438392865, "mean_first_positive_rank": 1.026, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 57.84846639241278, "median_score_margin": 63.093711376190186, "negative_margin_rate": 0.0255, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.5898566703417861, "recall1": 0.5898566703417861, "recall3": 0.6896361631753032, "recall5": 0.7464167585446527, "recall8": 0.8676957001102535, "evidence_count_in_top3": 0.6896361631753032, "hit3": 0.6896361631753032, "mrr": 0.6759606339151697, "ndcg3": 0.6479024598421029, "mean_first_positive_rank": 3.383131201764057, "all_evidence_in_top3": 0.6896361631753032, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.20369631854576997, "median_score_margin": 6.827788352966309, "negative_margin_rate": 0.4101433296582139, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999848195447, "unknown_mean_need_prob": 0.00018733273071117103, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 3.3919047850488803e-15, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "REPLAY-128 final": { "checkpoint": "checkpoints\\router_replay_v7_v2_128\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 8156921 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 8156921 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.9342146668738375, "single_query_latency_ms_p50": 0.8827999990899116, "single_query_latency_ms_p95": 1.4861000017845072, "queries_per_second": 1070.417790962649, "samples": 150, "batched_qps_64": 70303.40306999629, "batched_latency_ms_64": 0.014224062510947988, "batched_qps_256": 244326.0991071313, "batched_latency_ms_256": 0.004092890623041967 }, "cpu": { "single_query_latency_ms_mean": 0.3855180000149024, "single_query_latency_ms_p50": 0.36720000207424164, "single_query_latency_ms_p95": 0.5253999988781288, "queries_per_second": 2593.912605796213, "samples": 150, "batched_qps_64": 12253.306479886014, "batched_latency_ms_64": 0.08161062498857063, "batched_qps_256": 12008.169307919601, "batched_latency_ms_256": 0.08327664062335316 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9413944088918828, "recall1": 0.8857920736499383, "recall3": 0.9647748961490962, "recall5": 0.9714550353654429, "recall8": 0.9804086673402942, "evidence_count_in_top3": 1.0744919726058157, "hit3": 0.9673290670259347, "mrr": 0.957689277532011, "ndcg3": 0.9536787031890833, "mean_first_positive_rank": 1.34001347254968, "all_evidence_in_top3": 0.9622207252722578, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9545, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 40.14557690149532, "median_score_margin": 45.34479379653931, "negative_margin_rate": 0.05860559110811721, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999984274064, "unknown_mean_need_prob": 4.836221294362452e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 4.8406160974828055e-06, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.98225, "recall1": 0.92034375, "recall3": 0.99715625, "recall5": 0.99959375, "recall8": 0.9996875, "evidence_count_in_top3": 1.1193125, "hit3": 1.0, "mrr": 0.9911041666666667, "ndcg3": 0.9893498205729669, "mean_first_positive_rank": 1.017875, "all_evidence_in_top3": 0.9943125, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9545, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 44.73340179644042, "median_score_margin": 47.92497134208679, "negative_margin_rate": 0.01775, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999701977, "unknown_mean_need_prob": 6.379481702447893e-11, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.5875805506736965, "recall1": 0.5875805506736965, "recall3": 0.6731107205623902, "recall5": 0.7106033977738723, "recall8": 0.7996485061511424, "evidence_count_in_top3": 0.6731107205623902, "hit3": 0.6731107205623902, "mrr": 0.6633131807141506, "ndcg3": 0.6380925044585338, "mean_first_positive_rank": 4.295254833040421, "all_evidence_in_top3": 0.6731107205623902, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.2721888081208264, "median_score_margin": 8.391286849975586, "negative_margin_rate": 0.4124194493263035, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999838679872, "unknown_mean_need_prob": 0.0004654159529796483, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.4766355140186916, "recall1": 0.4766355140186916, "recall3": 0.7757009345794392, "recall5": 0.9252336448598131, "recall8": 0.9813084112149533, "evidence_count_in_top3": 0.7757009345794392, "hit3": 0.7757009345794392, "mrr": 0.6573319982198488, "ndcg3": 0.6543120031041451, "mean_first_positive_rank": 2.364485981308411, "all_evidence_in_top3": 0.7757009345794392, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -1.0887432181946586, "median_score_margin": -0.6999244689941406, "negative_margin_rate": 0.5233644859813084, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0004654159529796483, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.991, "recall1": 0.991, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9955, "ndcg3": 0.9966783677821431, "mean_first_positive_rank": 1.009, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 42.24227243242413, "median_score_margin": 42.77137851715088, "negative_margin_rate": 0.009, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.973, "recall1": 0.973, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9865, "ndcg3": 0.9900351033464293, "mean_first_positive_rank": 1.027, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 50.82861385926604, "median_score_margin": 54.98523235321045, "negative_margin_rate": 0.027, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.983, "recall1": 0.983, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9915, "ndcg3": 0.9937258058107148, "mean_first_positive_rank": 1.017, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 51.79758185644448, "median_score_margin": 55.65491724014282, "negative_margin_rate": 0.017, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.275862421441728e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.978, "recall1": 0.978, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9889166666666667, "ndcg3": 0.9918149897017863, "mean_first_positive_rank": 1.0225, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 47.034401270009575, "median_score_margin": 50.28937745094299, "negative_margin_rate": 0.022, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9905, "recall1": 0.49525, "recall3": 0.97725, "recall5": 0.99675, "recall8": 0.9975, "evidence_count_in_top3": 1.9545, "hit3": 1.0, "mrr": 0.99525, "ndcg3": 0.963831301989089, "mean_first_positive_rank": 1.0095, "all_evidence_in_top3": 0.9545, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.9545, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 46.103771095309405, "median_score_margin": 52.25391101837158, "negative_margin_rate": 0.0095, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999997615814, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.994, "recall1": 0.994, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.997, "ndcg3": 0.9977855785214288, "mean_first_positive_rank": 1.006, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 36.59333083114028, "median_score_margin": 35.444119691848755, "negative_margin_rate": 0.006, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.978, "recall1": 0.978, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9889166666666667, "ndcg3": 0.9918149897017863, "mean_first_positive_rank": 1.0225, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 35.780669214252384, "median_score_margin": 35.064393758773804, "negative_margin_rate": 0.022, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9705, "recall1": 0.9705, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98525, "ndcg3": 0.989112427730358, "mean_first_positive_rank": 1.0295, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 47.48657381267706, "median_score_margin": 51.48151856660843, "negative_margin_rate": 0.0295, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.5810363836824697, "recall1": 0.5810363836824697, "recall3": 0.679162072767365, "recall5": 0.7232635060639471, "recall8": 0.8103638368246968, "evidence_count_in_top3": 0.679162072767365, "hit3": 0.679162072767365, "mrr": 0.662960376675071, "ndcg3": 0.6390492224051051, "mean_first_positive_rank": 4.181367144432194, "all_evidence_in_top3": 0.679162072767365, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.32035381466873164, "median_score_margin": 7.441610336303711, "negative_margin_rate": 0.4189636163175303, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999848195447, "unknown_mean_need_prob": 0.00018733273071117103, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 3.3919047850488803e-15, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } } }