{ "V2-512 best(step1000)": { "checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_router_512\\router_best.pt", "router": { "kind": "router_v2", "step": 1000, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3 }, "parameters": 4741902, "checkpoint_bytes": 56922381 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048.0, "checkpoint_bytes": 56922381 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.8951400030249109, "single_query_latency_ms_p50": 0.8387999841943383, "single_query_latency_ms_p95": 1.2322000111453235, "queries_per_second": 1117.1436832459055, "samples": 150 }, "cpu": { "single_query_latency_ms_mean": 0.8213466650340706, "single_query_latency_ms_p50": 0.8019999950192869, "single_query_latency_ms_p95": 1.0353000252507627, "queries_per_second": 1217.512705135923, "samples": 150 } }, "metrics": { "episodes": 1940, "answerable_episodes": 1834, "unknown_episodes": 106, "top1": 0.6712104689203926, "recall1": 0.6712104689203926, "recall3": 0.8876772082878953, "recall5": 0.9492911668484187, "recall8": 0.9858233369683751, "mrr": 0.7908385671852008, "ndcg3": 0.7999554748364778, "mean_first_positive_rank": 1.841875681570338, "all_evidence_in_top3": 0.8865866957470011, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.9783505154639175, "hop_accuracy_answerable": 0.9967284623773174, "hop_under_prediction": 0.003271537622682661, "hop_over_prediction": 0.0, "mean_score_margin": 0.5880086265317487, "median_score_margin": 0.3476818799972534, "negative_margin_rate": 0.3287895310796074, "hop_confusion": { "true_0_pred_0": 70, "true_0_pred_1": 36, "true_0_pred_2": 0, "true_1_pred_0": 4, "true_1_pred_1": 1828, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.9847397770947952, "unknown_mean_need_prob": 0.2536155528318629, "thr0.30": { "need_precision": 0.9786096256684492, "need_recall": 0.9978189749182116, "need_f1": 0.9881209503239742, "specificity_unknown_refusal": 0.6226415094339622, "abstention_accuracy": 0.977319587628866, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.37735849056603776, "counts": { "tp": 1830, "tn": 66, "fp": 40, "fn": 4 } }, "thr0.40": { "need_precision": 0.9796573875802997, "need_recall": 0.9978189749182116, "need_f1": 0.9886547811993517, "specificity_unknown_refusal": 0.6415094339622641, "abstention_accuracy": 0.9783505154639175, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.3584905660377358, "counts": { "tp": 1830, "tn": 68, "fp": 38, "fn": 4 } }, "thr0.50": { "need_precision": 0.9807073954983923, "need_recall": 0.9978189749182116, "need_f1": 0.9891891891891892, "specificity_unknown_refusal": 0.660377358490566, "abstention_accuracy": 0.979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.33962264150943394, "counts": { "tp": 1830, "tn": 70, "fp": 36, "fn": 4 } }, "thr0.60": { "need_precision": 0.9902597402597403, "need_recall": 0.9978189749182116, "need_f1": 0.9940249864204237, "specificity_unknown_refusal": 0.8301886792452831, "abstention_accuracy": 0.988659793814433, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.16981132075471697, "counts": { "tp": 1830, "tn": 88, "fp": 18, "fn": 4 } }, "thr0.70": { "need_precision": 0.9989082969432315, "need_recall": 0.9978189749182116, "need_f1": 0.9983633387888707, "specificity_unknown_refusal": 0.9811320754716981, "abstention_accuracy": 0.9969072164948454, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.018867924528301886, "counts": { "tp": 1830, "tn": 104, "fp": 2, "fn": 4 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.0, "recall8": 0.0, "mrr": 0.0, "ndcg3": 0.0, "mean_first_positive_rank": 0.0, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.0, "median_score_margin": 0.0, "negative_margin_rate": 0.0, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": 0.0, "unknown_mean_need_prob": 0.046122895131702535, "thr0.50": { "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20, "answerable_episodes": 20, "unknown_episodes": 0, "top1": 0.4, "recall1": 0.4, "recall3": 0.8, "recall5": 1.0, "recall8": 1.05, "mrr": 0.6225574712643678, "ndcg3": 0.6086215664331643, "mean_first_positive_rank": 3.45, "all_evidence_in_top3": 0.7, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.7, "hop_accuracy_answerable": 0.7, "hop_under_prediction": 0.3, "hop_over_prediction": 0.0, "mean_score_margin": -0.2963485062122345, "median_score_margin": -0.25011467933654785, "negative_margin_rate": 0.6, "hop_confusion": { "true_1_pred_0": 4, "true_1_pred_1": 14, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.8251703456044197, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 0.8, "need_f1": 0.888888888888889, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8, "known_question_refusal_rate": 0.2, "unknown_question_read_rate": 0.0, "counts": { "tp": 16, "tn": 0, "fp": 0, "fn": 4 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.6988869361452841, "recall1": 0.6988869361452841, "recall3": 0.9004100761570006, "recall5": 0.9537199765670767, "recall8": 0.9859402460456942, "mrr": 0.8099543326670381, "ndcg3": 0.8196676424617168, "mean_first_positive_rank": 1.7492677211482133, "all_evidence_in_top3": 0.9004100761570006, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 0.9895833333333334, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.6516403993389798, "median_score_margin": 0.38489770889282227, "negative_margin_rate": 0.30111306385471587, "hop_confusion": { "true_0_pred_0": 3, "true_0_pred_1": 18, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9862654927744807, "unknown_mean_need_prob": 0.5697948407559168, "thr0.50": { "need_precision": 0.9895652173913043, "need_recall": 1.0, "need_f1": 0.9947552447552448, "specificity_unknown_refusal": 0.14285714285714285, "abstention_accuracy": 0.9895833333333334, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.8571428571428571, "counts": { "tp": 1707, "tn": 3, "fp": 18, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.2803738317757009, "recall1": 0.2803738317757009, "recall3": 0.7009345794392523, "recall5": 0.8691588785046729, "recall8": 0.9719626168224299, "mrr": 0.5173339899975414, "ndcg3": 0.5212452695260414, "mean_first_positive_rank": 3.0186915887850465, "all_evidence_in_top3": 0.7009345794392523, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 0.859375, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.26182589241277393, "median_score_margin": -0.27979564666748047, "negative_margin_rate": 0.719626168224299, "hop_confusion": { "true_0_pred_0": 3, "true_0_pred_1": 18, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 0.9902256833058651, "unknown_mean_need_prob": 0.5697948407559168, "thr0.50": { "need_precision": 0.856, "need_recall": 1.0, "need_f1": 0.9224137931034483, "specificity_unknown_refusal": 0.14285714285714285, "abstention_accuracy": 0.859375, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.8571428571428571, "counts": { "tp": 107, "tn": 3, "fp": 18, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.25075531005859375, "median_score_margin": 0.25075531005859375, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9954056143760681, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 1.0, "recall1": 1.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 1.0, "ndcg3": 1.0, "mean_first_positive_rank": 1.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 3.3609426021575928, "median_score_margin": 3.3609426021575928, "negative_margin_rate": 0.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.971575915813446, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 1.2087433338165283, "median_score_margin": 1.2087433338165283, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.92432501912117, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.5, "recall5": 2.0, "recall8": 2.0, "mrr": 0.75, "ndcg3": 0.6934264036172708, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": 10.865453004837036, "median_score_margin": 10.865453004837036, "negative_margin_rate": 0.5, "hop_confusion": { "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.9887627363204956, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 0.5, "recall5": 1.0, "recall8": 1.0, "mrr": 0.625, "ndcg3": 0.5, "mean_first_positive_rank": 2.5, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -1.6004958152770996, "median_score_margin": -1.6004958152770996, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9807772934436798, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.5, "recall5": 1.0, "recall8": 1.0, "mrr": 0.375, "ndcg3": 0.31546487678572877, "mean_first_positive_rank": 3.0, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": -3.42048117518425, "median_score_margin": -3.42048117518425, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_0": 2, "true_1_pred_1": 0 }, "known_mean_need_prob": 0.1579926460981369, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2 } } }, "paraphrase": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.5, "recall5": 0.5, "recall8": 1.0, "mrr": 0.3333333333333333, "ndcg3": 0.31546487678572877, "mean_first_positive_rank": 4.0, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -2.6525142192840576, "median_score_margin": -2.6525142192840576, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.99253049492836, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 1.0, "recall1": 1.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 1.0, "ndcg3": 1.0, "mean_first_positive_rank": 1.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 3.147550106048584, "median_score_margin": 3.147550106048584, "negative_margin_rate": 0.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9879558682441711, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.5, "ndcg3": 0.6309297535714575, "mean_first_positive_rank": 2.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.12020492553710938, "median_score_margin": -0.12020492553710938, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9997169077396393, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.6742006615214995, "recall1": 0.6742006615214995, "recall3": 0.8886438809261301, "recall5": 0.9487320837927232, "recall8": 0.9851157662624035, "mrr": 0.792693926566908, "ndcg3": 0.8020649997361834, "mean_first_positive_rank": 1.8241455347298787, "all_evidence_in_top3": 0.8886438809261301, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 0.98125, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.5977589808067649, "median_score_margin": 0.348813533782959, "negative_margin_rate": 0.32579933847850057, "hop_confusion": { "true_0_pred_0": 70, "true_0_pred_1": 36, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9864990872545568, "unknown_mean_need_prob": 0.2536155528318629, "thr0.50": { "need_precision": 0.9805405405405405, "need_recall": 1.0, "need_f1": 0.990174672489083, "specificity_unknown_refusal": 0.660377358490566, "abstention_accuracy": 0.98125, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.33962264150943394, "counts": { "tp": 1814, "tn": 70, "fp": 36, "fn": 0 } } }, "unknown_abstention": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.5, "recall8": 0.5, "mrr": 0.14224137931034483, "ndcg3": 0.0, "mean_first_positive_rank": 16.5, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": -14.003233283758163, "median_score_margin": -14.003233283758163, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_0": 2, "true_1_pred_1": 0 }, "known_mean_need_prob": 0.25266095995903015, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2 } } } }, "protocol_consistency_delta": { "hop_accuracy": 0.0, "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "abstention_accuracy": 0.0 } }, "V2-512 final(step100k)": { "checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_router_512\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3 }, "parameters": 4741902, "checkpoint_bytes": 18973433 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048.0, "checkpoint_bytes": 18973433 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.9346639970317483, "single_query_latency_ms_p50": 0.8856999920681119, "single_query_latency_ms_p95": 1.2600999907590449, "queries_per_second": 1069.9031985566385, "samples": 150 }, "cpu": { "single_query_latency_ms_mean": 0.8593879973826309, "single_query_latency_ms_p50": 0.8493000059388578, "single_query_latency_ms_p95": 1.121999986935407, "queries_per_second": 1163.6187648019518, "samples": 150 } }, "metrics": { "episodes": 1940, "answerable_episodes": 1834, "unknown_episodes": 106, "top1": 0.6063249727371864, "recall1": 0.6063249727371864, "recall3": 0.8500545256270448, "recall5": 0.9013086150490731, "recall8": 0.9487459105779716, "mrr": 0.7325518172051582, "ndcg3": 0.7456965349469862, "mean_first_positive_rank": 2.6052344601962925, "all_evidence_in_top3": 0.8489640130861504, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.9969072164948454, "hop_accuracy_answerable": 0.9967284623773174, "hop_under_prediction": 0.003271537622682661, "hop_over_prediction": 0.0, "mean_score_margin": 3.8836881184794065, "median_score_margin": 6.41892671585083, "negative_margin_rate": 0.3936750272628135, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 4, "true_1_pred_1": 1828, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.9977977412946947, "unknown_mean_need_prob": 0.01519268750541802, "thr0.30": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.0, "recall8": 0.0, "mrr": 0.0, "ndcg3": 0.0, "mean_first_positive_rank": 0.0, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.0, "median_score_margin": 0.0, "negative_margin_rate": 0.0, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": 0.0, "unknown_mean_need_prob": 8.623558289710172e-08, "thr0.50": { "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20, "answerable_episodes": 20, "unknown_episodes": 0, "top1": 0.35, "recall1": 0.35, "recall3": 0.8, "recall5": 0.9, "recall8": 0.9, "mrr": 0.5523787878787878, "ndcg3": 0.5633897980311373, "mean_first_positive_rank": 4.55, "all_evidence_in_top3": 0.7, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.7, "hop_accuracy_answerable": 0.7, "hop_under_prediction": 0.3, "hop_over_prediction": 0.0, "mean_score_margin": -12.90834836959839, "median_score_margin": -9.948488235473633, "negative_margin_rate": 0.65, "hop_confusion": { "true_1_pred_0": 4, "true_1_pred_1": 14, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.8000006015426834, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 0.8, "need_f1": 0.888888888888889, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8, "known_question_refusal_rate": 0.2, "unknown_question_read_rate": 0.0, "counts": { "tp": 16, "tn": 0, "fp": 0, "fn": 4 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.6256590509666081, "recall1": 0.6256590509666081, "recall3": 0.859402460456942, "recall5": 0.9015817223198594, "recall8": 0.9472759226713533, "mrr": 0.7458644125569885, "ndcg3": 0.7599420351849566, "mean_first_positive_rank": 2.5606326889279436, "all_evidence_in_top3": 0.859402460456942, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 5.392942665265922, "median_score_margin": 7.422682553529739, "negative_margin_rate": 0.37434094903339193, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.999977179556893, "unknown_mean_need_prob": 0.038343318011833445, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.34579439252336447, "recall1": 0.34579439252336447, "recall3": 0.7102803738317757, "recall5": 0.897196261682243, "recall8": 0.9813084112149533, "mrr": 0.5538495772140631, "ndcg3": 0.552510234312422, "mean_first_positive_rank": 2.953271028037383, "all_evidence_in_top3": 0.7102803738317757, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -17.0551603077171, "median_score_margin": -14.022430419921875, "negative_margin_rate": 0.6542056074766355, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.038343318011833445, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.41666666666666663, "ndcg3": 0.5654648767857288, "mean_first_positive_rank": 2.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -10.757410049438477, "median_score_margin": -10.757410049438477, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.6666666666666666, "ndcg3": 0.75, "mean_first_positive_rank": 2.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -12.85321569442749, "median_score_margin": -12.85321569442749, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9999998807907104, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 11.889404773712158, "median_score_margin": 11.889404773712158, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9999871253967285, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 1.0, "recall1": 1.0, "recall3": 1.5, "recall5": 2.0, "recall8": 2.0, "mrr": 1.0, "ndcg3": 0.8065735963827292, "mean_first_positive_rank": 1.0, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": 21.42271614074707, "median_score_margin": 21.42271614074707, "negative_margin_rate": 0.0, "hop_confusion": { "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.9999992251396179, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -24.237308502197266, "median_score_margin": -24.237308502197266, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.0, "recall8": 0.0, "mrr": 0.0787878787878788, "ndcg3": 0.0, "mean_first_positive_rank": 13.0, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": -33.50318145751953, "median_score_margin": -33.50318145751953, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_0": 2, "true_1_pred_1": 0 }, "known_mean_need_prob": 2.4256567030533915e-06, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2 } } }, "paraphrase": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.5, "recall8": 0.5, "mrr": 0.175, "ndcg3": 0.0, "mean_first_positive_rank": 7.0, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -41.73876094818115, "median_score_margin": -41.73876094818115, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 31.204745292663574, "median_score_margin": 31.204745292663574, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.41666666666666663, "ndcg3": 0.5654648767857288, "mean_first_positive_rank": 2.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -12.233808517456055, "median_score_margin": -12.233808517456055, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.6091510474090408, "recall1": 0.6091510474090408, "recall3": 0.850606394707828, "recall5": 0.9013230429988974, "recall8": 0.9492833517089305, "mrr": 0.7345382894138281, "ndcg3": 0.7477065320463893, "mean_first_positive_rank": 2.5837927232635063, "all_evidence_in_top3": 0.850606394707828, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 4.068826337752591, "median_score_margin": 6.489462748169899, "negative_margin_rate": 0.3908489525909592, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999785256359517, "unknown_mean_need_prob": 0.01519268750541802, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 0.5, "recall5": 0.5, "recall8": 0.5, "mrr": 0.52, "ndcg3": 0.5, "mean_first_positive_rank": 13.0, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": -58.27666473388672, "median_score_margin": -58.27666473388672, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_0": 2, "true_1_pred_1": 0 }, "known_mean_need_prob": 1.735844307404477e-05, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2 } } } }, "protocol_consistency_delta": { "hop_accuracy": 0.0, "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "abstention_accuracy": 0.0 } }, "XL-512 best": { "checkpoint": "H:\\Memory\\V2_dpskw\\checkpoints\\router_xl_512\\router_best.pt", "router": { "kind": "router_xl", "step": 3500, "arch_config": { "arch": "router_xl", "arch_version": 1, "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3, "encoder_layers": 2, "encoder_hidden": 512, "pair_blocks": 1, "pair_hidden": 512, "pair_expansion": 2, "pair_dropout": 0.05, "use_interaction": true, "policy_layers": 2, "policy_hidden": 512, "policy_dropout": 0.0, "learnable_cosine_scale": true }, "parameters": 7898127, "checkpoint_bytes": 99014505 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048.0, "checkpoint_bytes": 99014505 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.3101640006061643, "single_query_latency_ms_p50": 1.2462000013329089, "single_query_latency_ms_p95": 1.7039999947883189, "queries_per_second": 763.263224708767, "samples": 150 }, "cpu": { "single_query_latency_ms_mean": 1.719946669957911, "single_query_latency_ms_p50": 1.7377999611198902, "single_query_latency_ms_p95": 2.036199963185936, "queries_per_second": 581.4133760464044, "samples": 150 } }, "metrics": { "episodes": 1940, "answerable_episodes": 1834, "unknown_episodes": 106, "top1": 0.608505997818975, "recall1": 0.608505997818975, "recall3": 0.9296619411123228, "recall5": 0.9732824427480916, "recall8": 0.9929116684841875, "mrr": 0.759814160403073, "ndcg3": 0.792764395065411, "mean_first_positive_rank": 1.83206106870229, "all_evidence_in_top3": 0.9285714285714286, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.9938144329896907, "hop_accuracy_answerable": 0.9967284623773174, "hop_under_prediction": 0.003271537622682661, "hop_over_prediction": 0.0, "mean_score_margin": 6.709678303645345, "median_score_margin": 2.013867676258087, "negative_margin_rate": 0.3914940021810251, "hop_confusion": { "true_0_pred_0": 100, "true_0_pred_1": 6, "true_0_pred_2": 0, "true_1_pred_0": 4, "true_1_pred_1": 1828, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.9962027715833912, "unknown_mean_need_prob": 0.13497691851203716, "thr0.30": { "need_precision": 0.9859913793103449, "need_recall": 0.9978189749182116, "need_f1": 0.9918699186991871, "specificity_unknown_refusal": 0.7547169811320755, "abstention_accuracy": 0.9845360824742269, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.24528301886792453, "counts": { "tp": 1830, "tn": 80, "fp": 26, "fn": 4 } }, "thr0.40": { "need_precision": 0.9924078091106291, "need_recall": 0.9978189749182116, "need_f1": 0.9951060358890701, "specificity_unknown_refusal": 0.8679245283018868, "abstention_accuracy": 0.9907216494845361, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.1320754716981132, "counts": { "tp": 1830, "tn": 92, "fp": 14, "fn": 4 } }, "thr0.50": { "need_precision": 0.9945652173913043, "need_recall": 0.9978189749182116, "need_f1": 0.9961894393032119, "specificity_unknown_refusal": 0.9056603773584906, "abstention_accuracy": 0.9927835051546392, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.09433962264150944, "counts": { "tp": 1830, "tn": 96, "fp": 10, "fn": 4 } }, "thr0.60": { "need_precision": 0.9989082969432315, "need_recall": 0.9978189749182116, "need_f1": 0.9983633387888707, "specificity_unknown_refusal": 0.9811320754716981, "abstention_accuracy": 0.9969072164948454, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.018867924528301886, "counts": { "tp": 1830, "tn": 104, "fp": 2, "fn": 4 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.0, "recall8": 0.0, "mrr": 0.0, "ndcg3": 0.0, "mean_first_positive_rank": 0.0, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.0, "median_score_margin": 0.0, "negative_margin_rate": 0.0, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": 0.0, "unknown_mean_need_prob": 0.003423646269766323, "thr0.50": { "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20, "answerable_episodes": 20, "unknown_episodes": 0, "top1": 0.4, "recall1": 0.4, "recall3": 0.85, "recall5": 0.9, "recall8": 0.9, "mrr": 0.6083352272727273, "ndcg3": 0.6361540935691467, "mean_first_positive_rank": 6.15, "all_evidence_in_top3": 0.75, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.7, "hop_accuracy_answerable": 0.7, "hop_under_prediction": 0.3, "hop_over_prediction": 0.0, "mean_score_margin": -0.6135687932372094, "median_score_margin": -0.7090425491333008, "negative_margin_rate": 0.6, "hop_confusion": { "true_1_pred_0": 4, "true_1_pred_1": 14, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.8056919910013676, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 0.8, "need_f1": 0.888888888888889, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8, "known_question_refusal_rate": 0.2, "unknown_question_read_rate": 0.0, "counts": { "tp": 16, "tn": 0, "fp": 0, "fn": 4 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.6239015817223199, "recall1": 0.6239015817223199, "recall3": 0.9420035149384886, "recall5": 0.9806678383128296, "recall8": 0.9953134153485648, "mrr": 0.7716295930129063, "ndcg3": 0.8064999615386277, "mean_first_positive_rank": 1.7258347978910369, "all_evidence_in_top3": 0.9420035149384886, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 0.9982638888888888, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 7.258645884769051, "median_score_margin": 2.5352187156677246, "negative_margin_rate": 0.37609841827768015, "hop_confusion": { "true_0_pred_0": 18, "true_0_pred_1": 3, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9981995297581673, "unknown_mean_need_prob": 0.33543904764311655, "thr0.50": { "need_precision": 0.9970794392523364, "need_recall": 1.0, "need_f1": 0.9985375840889148, "specificity_unknown_refusal": 0.7619047619047619, "abstention_accuracy": 0.9971064814814815, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.23809523809523808, "counts": { "tp": 1707, "tn": 16, "fp": 5, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.40186915887850466, "recall1": 0.40186915887850466, "recall3": 0.7476635514018691, "recall5": 0.8691588785046729, "recall8": 0.9719626168224299, "mrr": 0.5996331809415921, "ndcg3": 0.6029101339452666, "mean_first_positive_rank": 2.7196261682242993, "all_evidence_in_top3": 0.7476635514018691, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 0.9765625, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.6793190705650878, "median_score_margin": -0.7896919250488281, "negative_margin_rate": 0.5981308411214953, "hop_confusion": { "true_0_pred_0": 18, "true_0_pred_1": 3, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 0.9999574389413138, "unknown_mean_need_prob": 0.33543904764311655, "thr0.50": { "need_precision": 0.9553571428571429, "need_recall": 1.0, "need_f1": 0.9771689497716896, "specificity_unknown_refusal": 0.7619047619047619, "abstention_accuracy": 0.9609375, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.23809523809523808, "counts": { "tp": 107, "tn": 16, "fp": 5, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 1.90769624710083, "median_score_margin": 1.90769624710083, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9999270439147949, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 1.0, "recall1": 1.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 1.0, "ndcg3": 1.0, "mean_first_positive_rank": 1.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 4.908981084823608, "median_score_margin": 4.908981084823608, "negative_margin_rate": 0.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9988398551940918, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 7.163186550140381, "median_score_margin": 7.163186550140381, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9931761622428894, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.5, "recall5": 2.0, "recall8": 2.0, "mrr": 0.75, "ndcg3": 0.6532867981913646, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": 4.611967086791992, "median_score_margin": 4.611967086791992, "negative_margin_rate": 0.5, "hop_confusion": { "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.998505711555481, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -1.0701360702514648, "median_score_margin": -1.0701360702514648, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9997996389865875, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.0, "recall8": 0.0, "mrr": 0.045, "ndcg3": 0.0, "mean_first_positive_rank": 22.5, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": -13.473207950592041, "median_score_margin": -13.473207950592041, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_0": 2, "true_1_pred_1": 0 }, "known_mean_need_prob": 0.02726295404136181, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2 } } }, "paraphrase": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 5.622410535812378, "median_score_margin": 5.622410535812378, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9998534619808197, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 5.728376388549805, "median_score_margin": 5.728376388549805, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9995926022529602, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.5, "ndcg3": 0.6309297535714575, "mean_first_positive_rank": 2.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.9848322868347168, "median_score_margin": -0.9848322868347168, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9999998807907104, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.6108048511576626, "recall1": 0.6108048511576626, "recall3": 0.9305402425578831, "recall5": 0.974090407938258, "recall8": 0.9939360529217199, "mrr": 0.7614842699193943, "ndcg3": 0.7944910797566598, "mean_first_positive_rank": 1.7844542447629548, "all_evidence_in_top3": 0.9305402425578831, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 0.996875, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 6.790419726984734, "median_score_margin": 2.0423730704933405, "negative_margin_rate": 0.38919514884233736, "hop_confusion": { "true_0_pred_0": 100, "true_0_pred_1": 6, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9983032212039207, "unknown_mean_need_prob": 0.13497691851203716, "thr0.50": { "need_precision": 0.9945175438596491, "need_recall": 1.0, "need_f1": 0.9972512369433755, "specificity_unknown_refusal": 0.9056603773584906, "abstention_accuracy": 0.9947916666666666, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.09433962264150944, "counts": { "tp": 1814, "tn": 96, "fp": 10, "fn": 0 } } }, "unknown_abstention": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.0, "recall8": 0.0, "mrr": 0.03835227272727273, "ndcg3": 0.0, "mean_first_positive_rank": 27.0, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": -20.550129517912865, "median_score_margin": -20.550129517912865, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_0": 2, "true_1_pred_1": 0 }, "known_mean_need_prob": 0.03996259905397892, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2 } } } }, "protocol_consistency_delta": { "hop_accuracy": 0.0, "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "abstention_accuracy": 0.0 } }, "XL-512 final(step100k)": { "checkpoint": "H:\\Memory\\V2_dpskw\\checkpoints\\router_xl_512\\memory_router_xl.pt", "router": { "kind": "router_xl", "step": null, "arch_config": { "arch": "router_xl", "arch_version": 1, "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3, "encoder_layers": 2, "encoder_hidden": 512, "pair_blocks": 1, "pair_hidden": 512, "pair_expansion": 2, "pair_dropout": 0.05, "use_interaction": true, "policy_layers": 2, "policy_hidden": 512, "policy_dropout": 0.0, "learnable_cosine_scale": true }, "parameters": 7898127, "checkpoint_bytes": 35802155 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048.0, "checkpoint_bytes": 35802155 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.2458240000220637, "single_query_latency_ms_p50": 1.1972999782301486, "single_query_latency_ms_p95": 1.5769999590702355, "queries_per_second": 802.6815986706708, "samples": 150 }, "cpu": { "single_query_latency_ms_mean": 1.7858959986673046, "single_query_latency_ms_p50": 1.7789999837987125, "single_query_latency_ms_p95": 2.090599969960749, "queries_per_second": 559.9430206161128, "samples": 150 } }, "metrics": { "episodes": 1940, "answerable_episodes": 1834, "unknown_episodes": 106, "top1": 0.5436205016357688, "recall1": 0.5436205016357688, "recall3": 0.8533260632497274, "recall5": 0.9193020719738277, "recall8": 0.9596510359869138, "mrr": 0.6950444145765433, "ndcg3": 0.7170573814087626, "mean_first_positive_rank": 2.6161395856052345, "all_evidence_in_top3": 0.8522355507088332, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.9969072164948454, "hop_accuracy_answerable": 0.9967284623773174, "hop_under_prediction": 0.003271537622682661, "hop_over_prediction": 0.0, "mean_score_margin": 25.375655539715694, "median_score_margin": 5.385736465454102, "negative_margin_rate": 0.4563794983642312, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 4, "true_1_pred_1": 1828, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.997793581256602, "unknown_mean_need_prob": 0.012418151622086199, "thr0.30": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.0, "recall8": 0.0, "mrr": 0.0, "ndcg3": 0.0, "mean_first_positive_rank": 0.0, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.0, "median_score_margin": 0.0, "negative_margin_rate": 0.0, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": 0.0, "unknown_mean_need_prob": 8.598426478830835e-08, "thr0.50": { "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20, "answerable_episodes": 20, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 0.8, "recall5": 0.8, "recall8": 0.85, "mrr": 0.6361078431372549, "ndcg3": 0.6455285910760186, "mean_first_positive_rank": 6.8, "all_evidence_in_top3": 0.7, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.7, "hop_accuracy_answerable": 0.7, "hop_under_prediction": 0.3, "hop_over_prediction": 0.0, "mean_score_margin": 3.729068946838379, "median_score_margin": 0.8376903533935547, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_0": 4, "true_1_pred_1": 14, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.8000009355226269, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 0.8, "need_f1": 0.888888888888889, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.8, "known_question_refusal_rate": 0.2, "unknown_question_read_rate": 0.0, "counts": { "tp": 16, "tn": 0, "fp": 0, "fn": 4 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.5512595196250732, "recall1": 0.5512595196250732, "recall3": 0.8576449912126538, "recall5": 0.9203280609256005, "recall8": 0.9601640304628002, "mrr": 0.7002585141169417, "ndcg3": 0.7229373582956774, "mean_first_positive_rank": 2.5735207967193907, "all_evidence_in_top3": 0.8576449912126538, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 27.063399425814147, "median_score_margin": 6.595245361328125, "negative_margin_rate": 0.44874048037492675, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999727061008528, "unknown_mean_need_prob": 0.03134091830829026, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.42990654205607476, "recall1": 0.42990654205607476, "recall3": 0.794392523364486, "recall5": 0.9252336448598131, "recall8": 0.9719626168224299, "mrr": 0.62287865301884, "ndcg3": 0.6366223838451323, "mean_first_positive_rank": 2.514018691588785, "all_evidence_in_top3": 0.794392523364486, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 2.4967108508136784, "median_score_margin": -7.090728759765625, "negative_margin_rate": 0.5700934579439252, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.03134091830829026, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 1.0507621765136719, "median_score_margin": 1.0507621765136719, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 1.0, "recall1": 1.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 1.0, "ndcg3": 1.0, "mean_first_positive_rank": 1.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 106.49523544311523, "median_score_margin": 106.49523544311523, "negative_margin_rate": 0.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 1.0, "recall1": 1.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 1.0, "ndcg3": 1.0, "mean_first_positive_rank": 1.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 86.62086868286133, "median_score_margin": 86.62086868286133, "negative_margin_rate": 0.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9999902844429016, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.5, "recall5": 1.5, "recall8": 2.0, "mrr": 0.75, "ndcg3": 0.6934264036172708, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": 53.472116470336914, "median_score_margin": 53.472116470336914, "negative_margin_rate": 0.5, "hop_confusion": { "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.9999991655349731, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.5, "recall5": 0.5, "recall8": 0.5, "mrr": 0.27, "ndcg3": 0.31546487678572877, "mean_first_positive_rank": 13.5, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -53.50970458984375, "median_score_margin": -53.50970458984375, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.0, "recall8": 0.0, "mrr": 0.04941176470588235, "ndcg3": 0.0, "mean_first_positive_rank": 21.0, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": -134.2181854248047, "median_score_margin": -134.2181854248047, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_0": 2, "true_1_pred_1": 0 }, "known_mean_need_prob": 2.6863836524171347e-06, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2 } } }, "paraphrase": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 7.193164825439453, "median_score_margin": 7.193164825439453, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 1.0, "recall1": 1.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 1.0, "ndcg3": 1.0, "mean_first_positive_rank": 1.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 108.70876693725586, "median_score_margin": 108.70876693725586, "negative_margin_rate": 0.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -7.420085906982422, "median_score_margin": -7.420085906982422, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.5441014332965821, "recall1": 0.5441014332965821, "recall3": 0.8539140022050716, "recall5": 0.9206174200661521, "recall8": 0.9608599779492834, "mrr": 0.695694211395058, "ndcg3": 0.7178460119526738, "mean_first_positive_rank": 2.570011025358324, "all_evidence_in_top3": 0.8539140022050716, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 25.61431691339681, "median_score_margin": 5.385736465454102, "negative_margin_rate": 0.4558985667034179, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999743160496999, "unknown_mean_need_prob": 0.012418151622086199, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.0, "recall8": 0.0, "mrr": 0.041666666666666664, "ndcg3": 0.0, "mean_first_positive_rank": 24.5, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": 0.0, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": -131.1022491455078, "median_score_margin": -131.1022491455078, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_0": 2, "true_1_pred_1": 0 }, "known_mean_need_prob": 1.7218864741153084e-05, "unknown_mean_need_prob": 0.0, "thr0.50": { "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "specificity_unknown_refusal": 0.0, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2 } } } }, "protocol_consistency_delta": { "hop_accuracy": 0.0, "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "abstention_accuracy": 0.0 } } }