{ "V2-512 best(step1000)": { "checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_router_512\\router_best.pt", "router": { "kind": "router_v2", "step": 1000, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3 }, "parameters": 4741902, "checkpoint_bytes": 56922381 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048.0, "checkpoint_bytes": 56922381 }, "latency": { "cpu": { "single_query_latency_ms_mean": 1.2766800005920231, "single_query_latency_ms_p50": 1.2820999836549163, "single_query_latency_ms_p95": 1.3297999976202846, "queries_per_second": 783.2816363820841, "samples": 5 } }, "metrics": { "episodes": 1940, "answerable_episodes": 1834, "unknown_episodes": 106, "top1": 0.6712104689203926, "recall1": 0.6712104689203926, "recall3": 0.8876772082878953, "recall5": 0.9492911668484187, "recall8": 0.9858233369683751, "mrr": 0.7908385671852008, "ndcg3": 0.7999554748364778, "mean_first_positive_rank": 1.841875681570338, "all_evidence_in_top3": 0.8865866957470011, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.9783505154639175, "hop_accuracy_answerable": 0.9967284623773174, "hop_under_prediction": 0.003271537622682661, "hop_over_prediction": 0.0, "mean_score_margin": 0.5880087030850424, "median_score_margin": 0.347680926322937, "negative_margin_rate": 0.3287895310796074, "hop_confusion": { "true_0_pred_0": 70, "true_0_pred_1": 36, "true_0_pred_2": 0, "true_1_pred_0": 4, "true_1_pred_1": 1828, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.9847397789147844, "unknown_mean_need_prob": 0.2536155520674755, "thr0.30": { "need_precision": 0.9786096256684492, "need_recall": 0.9978189749182116, "need_f1": 0.9881209503239742, "specificity_unknown_refusal": 0.6226415094339622, "abstention_accuracy": 0.977319587628866, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.37735849056603776, "counts": { "tp": 1830, "tn": 66, "fp": 40, "fn": 4 } }, "thr0.40": { "need_precision": 0.9796573875802997, "need_recall": 0.9978189749182116, "need_f1": 0.9886547811993517, "specificity_unknown_refusal": 0.6415094339622641, "abstention_accuracy": 0.9783505154639175, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.3584905660377358, "counts": { "tp": 1830, "tn": 68, "fp": 38, "fn": 4 } }, "thr0.50": { "need_precision": 0.9807073954983923, "need_recall": 0.9978189749182116, "need_f1": 0.9891891891891892, "specificity_unknown_refusal": 0.660377358490566, "abstention_accuracy": 0.979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.33962264150943394, "counts": { "tp": 1830, "tn": 70, "fp": 36, "fn": 4 } }, "thr0.60": { "need_precision": 0.9902597402597403, "need_recall": 0.9978189749182116, "need_f1": 0.9940249864204237, "specificity_unknown_refusal": 0.8301886792452831, "abstention_accuracy": 0.988659793814433, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.16981132075471697, "counts": { "tp": 1830, "tn": 88, "fp": 18, "fn": 4 } }, "thr0.70": { "need_precision": 0.9989082969432315, "need_recall": 0.9978189749182116, "need_f1": 0.9983633387888707, "specificity_unknown_refusal": 0.9811320754716981, "abstention_accuracy": 0.9969072164948454, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.018867924528301886, "counts": { "tp": 1830, "tn": 104, "fp": 2, "fn": 4 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.04612289386568591, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20, "answerable_episodes": 20, "unknown_episodes": 0, "top1": 0.4, "recall1": 0.4, "recall3": 0.8, "recall5": 1.0, "recall8": 1.05, "mrr": 0.6225574712643678, "ndcg3": 0.6086215664331643, "mean_first_positive_rank": 3.45, "all_evidence_in_top3": 0.7, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.7, "hop_accuracy_answerable": 0.7, "hop_under_prediction": 0.3, "hop_over_prediction": 0.0, "mean_score_margin": -0.29634838700294497, "median_score_margin": -0.25011515617370605, "negative_margin_rate": 0.6, "hop_confusion": { "true_1_pred_0": 4, "true_1_pred_1": 14, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.8251703515648842, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 0.8, "need_f1": 0.888888888888889, "specificity_unknown_refusal": null, "abstention_accuracy": 0.8, "known_question_refusal_rate": 0.2, "unknown_question_read_rate": null, "counts": { "tp": 16, "tn": 0, "fp": 0, "fn": 4 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.6988869361452841, "recall1": 0.6988869361452841, "recall3": 0.9004100761570006, "recall5": 0.9537199765670767, "recall8": 0.9859402460456942, "mrr": 0.8099543326670381, "ndcg3": 0.8196676424617168, "mean_first_positive_rank": 1.7492677211482133, "all_evidence_in_top3": 0.9004100761570006, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.9895833333333334, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.6516404748137541, "median_score_margin": 0.38489866256713867, "negative_margin_rate": 0.30111306385471587, "hop_confusion": { "true_0_pred_0": 3, "true_0_pred_1": 18, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9862654947298761, "unknown_mean_need_prob": 0.5697948407559168, "thr0.50": { "need_precision": 0.9895652173913043, "need_recall": 1.0, "need_f1": 0.9947552447552448, "specificity_unknown_refusal": 0.14285714285714285, "abstention_accuracy": 0.9895833333333334, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.8571428571428571, "counts": { "tp": 1707, "tn": 3, "fp": 18, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.2803738317757009, "recall1": 0.2803738317757009, "recall3": 0.7009345794392523, "recall5": 0.8691588785046729, "recall8": 0.9719626168224299, "mrr": 0.5173339899975414, "ndcg3": 0.5212452695260414, "mean_first_positive_rank": 3.0186915887850465, "all_evidence_in_top3": 0.7009345794392523, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.859375, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.26182580662664967, "median_score_margin": -0.27979564666748047, "negative_margin_rate": 0.719626168224299, "hop_confusion": { "true_0_pred_0": 3, "true_0_pred_1": 18, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 0.9902256821917597, "unknown_mean_need_prob": 0.5697948407559168, "thr0.50": { "need_precision": 0.856, "need_recall": 1.0, "need_f1": 0.9224137931034483, "specificity_unknown_refusal": 0.14285714285714285, "abstention_accuracy": 0.859375, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.8571428571428571, "counts": { "tp": 107, "tn": 3, "fp": 18, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.25075769424438477, "median_score_margin": 0.25075769424438477, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9954056441783905, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 1.0, "recall1": 1.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 1.0, "ndcg3": 1.0, "mean_first_positive_rank": 1.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 3.3609426021575928, "median_score_margin": 3.3609426021575928, "negative_margin_rate": 0.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.971575915813446, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 1.208745002746582, "median_score_margin": 1.208745002746582, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.92432501912117, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.5, "recall5": 2.0, "recall8": 2.0, "mrr": 0.75, "ndcg3": 0.6934264036172708, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": 10.865452527999878, "median_score_margin": 10.865452527999878, "negative_margin_rate": 0.5, "hop_confusion": { "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.9887627363204956, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 0.5, "recall5": 1.0, "recall8": 1.0, "mrr": 0.625, "ndcg3": 0.5, "mean_first_positive_rank": 2.5, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -1.6004953384399414, "median_score_margin": -1.6004953384399414, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9807772934436798, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.5, "recall5": 1.0, "recall8": 1.0, "mrr": 0.375, "ndcg3": 0.31546487678572877, "mean_first_positive_rank": 3.0, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": -3.4204822182655334, "median_score_margin": -3.4204822182655334, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_0": 2, "true_1_pred_1": 0 }, "known_mean_need_prob": 0.1579926311969757, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": null, "need_recall": 0.0, "need_f1": null, "specificity_unknown_refusal": null, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": null, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2 } } }, "paraphrase": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.5, "recall5": 0.5, "recall8": 1.0, "mrr": 0.3333333333333333, "ndcg3": 0.31546487678572877, "mean_first_positive_rank": 4.0, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -2.652513027191162, "median_score_margin": -2.652513027191162, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.99253049492836, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 1.0, "recall1": 1.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 1.0, "ndcg3": 1.0, "mean_first_positive_rank": 1.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 3.1475493907928467, "median_score_margin": 3.1475493907928467, "negative_margin_rate": 0.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9879558682441711, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.5, "ndcg3": 0.6309297535714575, "mean_first_positive_rank": 2.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.12020683288574219, "median_score_margin": -0.12020683288574219, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9997169077396393, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.6742006615214995, "recall1": 0.6742006615214995, "recall3": 0.8886438809261301, "recall5": 0.9487320837927232, "recall8": 0.9851157662624035, "mrr": 0.792693926566908, "ndcg3": 0.8020649997361834, "mean_first_positive_rank": 1.8241455347298787, "all_evidence_in_top3": 0.8886438809261301, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.98125, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.5977590568897612, "median_score_margin": 0.3488118648529053, "negative_margin_rate": 0.32579933847850057, "hop_confusion": { "true_0_pred_0": 70, "true_0_pred_1": 36, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9864990890288957, "unknown_mean_need_prob": 0.2536155520674755, "thr0.50": { "need_precision": 0.9805405405405405, "need_recall": 1.0, "need_f1": 0.990174672489083, "specificity_unknown_refusal": 0.660377358490566, "abstention_accuracy": 0.98125, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.33962264150943394, "counts": { "tp": 1814, "tn": 70, "fp": 36, "fn": 0 } } }, "unknown_abstention": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.5, "recall8": 0.5, "mrr": 0.14224137931034483, "ndcg3": 0.0, "mean_first_positive_rank": 16.5, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": -14.003233671188354, "median_score_margin": -14.003233671188354, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_0": 2, "true_1_pred_1": 0 }, "known_mean_need_prob": 0.25266100466251373, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": null, "need_recall": 0.0, "need_f1": null, "specificity_unknown_refusal": null, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": null, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2 } } } }, "protocol_consistency_delta": { "hop_accuracy": 0.0, "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "abstention_accuracy": 0.0 } }, "V2-512 final(step100k)": { "checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_router_512\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3 }, "parameters": 4741902, "checkpoint_bytes": 18973433 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048.0, "checkpoint_bytes": 18973433 }, "latency": { "cpu": { "single_query_latency_ms_mean": 1.1631599976681173, "single_query_latency_ms_p50": 1.1254000128246844, "single_query_latency_ms_p95": 1.3188999728299677, "queries_per_second": 859.7269524440167, "samples": 5 } }, "metrics": { "episodes": 1940, "answerable_episodes": 1834, "unknown_episodes": 106, "top1": 0.6063249727371864, "recall1": 0.6063249727371864, "recall3": 0.8500545256270448, "recall5": 0.9013086150490731, "recall8": 0.9487459105779716, "mrr": 0.7325518172051582, "ndcg3": 0.7456965349469862, "mean_first_positive_rank": 2.6052344601962925, "all_evidence_in_top3": 0.8489640130861504, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.9969072164948454, "hop_accuracy_answerable": 0.9967284623773174, "hop_under_prediction": 0.003271537622682661, "hop_over_prediction": 0.0, "mean_score_margin": 3.8836888835735968, "median_score_margin": 6.418914318084717, "negative_margin_rate": 0.3936750272628135, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 4, "true_1_pred_1": 1828, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.9977977412946917, "unknown_mean_need_prob": 0.015192701093468172, "thr0.30": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 8.623557193993773e-08, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20, "answerable_episodes": 20, "unknown_episodes": 0, "top1": 0.35, "recall1": 0.35, "recall3": 0.8, "recall5": 0.9, "recall8": 0.9, "mrr": 0.5523787878787878, "ndcg3": 0.5633897980311373, "mean_first_positive_rank": 4.55, "all_evidence_in_top3": 0.7, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.7, "hop_accuracy_answerable": 0.7, "hop_under_prediction": 0.3, "hop_over_prediction": 0.0, "mean_score_margin": -12.90834379196167, "median_score_margin": -9.948484897613525, "negative_margin_rate": 0.65, "hop_confusion": { "true_1_pred_0": 4, "true_1_pred_1": 14, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.8000006015424048, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 0.8, "need_f1": 0.888888888888889, "specificity_unknown_refusal": null, "abstention_accuracy": 0.8, "known_question_refusal_rate": 0.2, "unknown_question_read_rate": null, "counts": { "tp": 16, "tn": 0, "fp": 0, "fn": 4 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.6256590509666081, "recall1": 0.6256590509666081, "recall3": 0.859402460456942, "recall5": 0.9015817223198594, "recall8": 0.9472759226713533, "mrr": 0.7458644125569885, "ndcg3": 0.7599420351849566, "mean_first_positive_rank": 2.5606326889279436, "all_evidence_in_top3": 0.859402460456942, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 5.392943512301349, "median_score_margin": 7.422684520483017, "negative_margin_rate": 0.37434094903339193, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.999977179556893, "unknown_mean_need_prob": 0.038343352305500525, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.34579439252336447, "recall1": 0.34579439252336447, "recall3": 0.7102803738317757, "recall5": 0.897196261682243, "recall8": 0.9813084112149533, "mrr": 0.5538495772140631, "ndcg3": 0.552510234312422, "mean_first_positive_rank": 2.953271028037383, "all_evidence_in_top3": 0.7102803738317757, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -17.055161562478432, "median_score_margin": -14.022438049316406, "negative_margin_rate": 0.6542056074766355, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.038343352305500525, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.41666666666666663, "ndcg3": 0.5654648767857288, "mean_first_positive_rank": 2.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -10.757413864135742, "median_score_margin": -10.757413864135742, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.6666666666666666, "ndcg3": 0.75, "mean_first_positive_rank": 2.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -12.8532075881958, "median_score_margin": -12.8532075881958, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9999998807907104, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 11.88942003250122, "median_score_margin": 11.88942003250122, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9999871253967285, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 1.0, "recall1": 1.0, "recall3": 1.5, "recall5": 2.0, "recall8": 2.0, "mrr": 1.0, "ndcg3": 0.8065735963827292, "mean_first_positive_rank": 1.0, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": 21.42270851135254, "median_score_margin": 21.42270851135254, "negative_margin_rate": 0.0, "hop_confusion": { "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.9999992251396179, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -24.237293243408203, "median_score_margin": -24.237293243408203, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.0, "recall8": 0.0, "mrr": 0.0787878787878788, "ndcg3": 0.0, "mean_first_positive_rank": 13.0, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": -33.503143310546875, "median_score_margin": -33.503143310546875, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_0": 2, "true_1_pred_1": 0 }, "known_mean_need_prob": 2.42565528196792e-06, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": null, "need_recall": 0.0, "need_f1": null, "specificity_unknown_refusal": null, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": null, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2 } } }, "paraphrase": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.5, "recall8": 0.5, "mrr": 0.175, "ndcg3": 0.0, "mean_first_positive_rank": 7.0, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -41.73877143859863, "median_score_margin": -41.73877143859863, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 31.204744338989258, "median_score_margin": 31.204744338989258, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.41666666666666663, "ndcg3": 0.5654648767857288, "mean_first_positive_rank": 2.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -12.233797550201416, "median_score_margin": -12.233797550201416, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.6091510474090408, "recall1": 0.6091510474090408, "recall3": 0.850606394707828, "recall5": 0.9013230429988974, "recall8": 0.9492833517089305, "mrr": 0.7345382894138281, "ndcg3": 0.7477065320463893, "mean_first_positive_rank": 2.5837927232635063, "all_evidence_in_top3": 0.850606394707828, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 4.068827060812134, "median_score_margin": 6.489464446902275, "negative_margin_rate": 0.3908489525909592, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999785256359517, "unknown_mean_need_prob": 0.015192701093468172, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 0.5, "recall5": 0.5, "recall8": 0.5, "mrr": 0.52, "ndcg3": 0.5, "mean_first_positive_rank": 13.0, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": -58.27668380737305, "median_score_margin": -58.27668380737305, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_0": 2, "true_1_pred_1": 0 }, "known_mean_need_prob": 1.735844170980272e-05, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": null, "need_recall": 0.0, "need_f1": null, "specificity_unknown_refusal": null, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": null, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2 } } } }, "protocol_consistency_delta": { "hop_accuracy": 0.0, "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "abstention_accuracy": 0.0 } }, "XL-512 best": { "checkpoint": "H:\\Memory\\V2_dpskw\\checkpoints\\router_xl_512\\router_best.pt", "router": { "kind": "router_xl", "step": 3500, "arch_config": { "arch": "router_xl", "arch_version": 1, "hidden_size": 2560, "router_dim": 512, "num_heads": 8, "max_hops": 3, "encoder_layers": 2, "encoder_hidden": 512, "pair_blocks": 1, "pair_hidden": 512, "pair_expansion": 2, "pair_dropout": 0.05, "use_interaction": true, "policy_layers": 2, "policy_hidden": 512, "policy_dropout": 0.0, "learnable_cosine_scale": true }, "parameters": 7898127, "checkpoint_bytes": 99014505 }, "storage": { "address_bytes_per_record": 2048, "address_mb_per_1m_records": 2048.0, "checkpoint_bytes": 99014505 }, "latency": { "cpu": { "single_query_latency_ms_mean": 2.5489200139418244, "single_query_latency_ms_p50": 2.477100002579391, "single_query_latency_ms_p95": 3.3594000269658864, "queries_per_second": 392.323020938398, "samples": 5 } }, "metrics": { "episodes": 1940, "answerable_episodes": 1834, "unknown_episodes": 106, "top1": 0.608505997818975, "recall1": 0.608505997818975, "recall3": 0.9296619411123228, "recall5": 0.9732824427480916, "recall8": 0.9929116684841875, "mrr": 0.759814160403073, "ndcg3": 0.792764395065411, "mean_first_positive_rank": 1.83206106870229, "all_evidence_in_top3": 0.9285714285714286, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.9938144329896907, "hop_accuracy_answerable": 0.9967284623773174, "hop_under_prediction": 0.003271537622682661, "hop_over_prediction": 0.0, "mean_score_margin": 6.709678291043545, "median_score_margin": 2.0138705372810364, "negative_margin_rate": 0.3914940021810251, "hop_confusion": { "true_0_pred_0": 100, "true_0_pred_1": 6, "true_0_pred_2": 0, "true_1_pred_0": 4, "true_1_pred_1": 1828, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.9962027715823756, "unknown_mean_need_prob": 0.13497692132687178, "thr0.30": { "need_precision": 0.9859913793103449, "need_recall": 0.9978189749182116, "need_f1": 0.9918699186991871, "specificity_unknown_refusal": 0.7547169811320755, "abstention_accuracy": 0.9845360824742269, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.24528301886792453, "counts": { "tp": 1830, "tn": 80, "fp": 26, "fn": 4 } }, "thr0.40": { "need_precision": 0.9924078091106291, "need_recall": 0.9978189749182116, "need_f1": 0.9951060358890701, "specificity_unknown_refusal": 0.8679245283018868, "abstention_accuracy": 0.9907216494845361, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.1320754716981132, "counts": { "tp": 1830, "tn": 92, "fp": 14, "fn": 4 } }, "thr0.50": { "need_precision": 0.9945652173913043, "need_recall": 0.9978189749182116, "need_f1": 0.9961894393032119, "specificity_unknown_refusal": 0.9056603773584906, "abstention_accuracy": 0.9927835051546392, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.09433962264150944, "counts": { "tp": 1830, "tn": 96, "fp": 10, "fn": 4 } }, "thr0.60": { "need_precision": 0.9989082969432315, "need_recall": 0.9978189749182116, "need_f1": 0.9983633387888707, "specificity_unknown_refusal": 0.9811320754716981, "abstention_accuracy": 0.9969072164948454, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.018867924528301886, "counts": { "tp": 1830, "tn": 104, "fp": 2, "fn": 4 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 0.9978189749182116, "need_f1": 0.9989082969432315, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9979381443298969, "known_question_refusal_rate": 0.0021810250817884407, "unknown_question_read_rate": 0.0, "counts": { "tp": 1830, "tn": 106, "fp": 0, "fn": 4 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 0.003423646275223291, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20, "answerable_episodes": 20, "unknown_episodes": 0, "top1": 0.4, "recall1": 0.4, "recall3": 0.85, "recall5": 0.9, "recall8": 0.9, "mrr": 0.6083352272727273, "ndcg3": 0.6361540935691467, "mean_first_positive_rank": 6.15, "all_evidence_in_top3": 0.75, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.7, "hop_accuracy_answerable": 0.7, "hop_under_prediction": 0.3, "hop_over_prediction": 0.0, "mean_score_margin": -0.6135697185993194, "median_score_margin": -0.7090378999710083, "negative_margin_rate": 0.6, "hop_confusion": { "true_1_pred_0": 4, "true_1_pred_1": 14, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.8056919909082353, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 0.8, "need_f1": 0.888888888888889, "specificity_unknown_refusal": null, "abstention_accuracy": 0.8, "known_question_refusal_rate": 0.2, "unknown_question_read_rate": null, "counts": { "tp": 16, "tn": 0, "fp": 0, "fn": 4 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.6239015817223199, "recall1": 0.6239015817223199, "recall3": 0.9420035149384886, "recall5": 0.9806678383128296, "recall8": 0.9953134153485648, "mrr": 0.7716295930129063, "ndcg3": 0.8064999615386277, "mean_first_positive_rank": 1.7258347978910369, "all_evidence_in_top3": 0.9420035149384886, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.9982638888888888, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 7.258645896374848, "median_score_margin": 2.5352253913879395, "negative_margin_rate": 0.37609841827768015, "hop_confusion": { "true_0_pred_0": 18, "true_0_pred_1": 3, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9981995297581673, "unknown_mean_need_prob": 0.33543905473890756, "thr0.50": { "need_precision": 0.9970794392523364, "need_recall": 1.0, "need_f1": 0.9985375840889148, "specificity_unknown_refusal": 0.7619047619047619, "abstention_accuracy": 0.9971064814814815, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.23809523809523808, "counts": { "tp": 1707, "tn": 16, "fp": 5, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.40186915887850466, "recall1": 0.40186915887850466, "recall3": 0.7476635514018691, "recall5": 0.8691588785046729, "recall8": 0.9719626168224299, "mrr": 0.5996331809415921, "ndcg3": 0.6029101339452666, "mean_first_positive_rank": 2.7196261682242993, "all_evidence_in_top3": 0.7476635514018691, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.9765625, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.6793192987478225, "median_score_margin": -0.7896833419799805, "negative_margin_rate": 0.5981308411214953, "hop_confusion": { "true_0_pred_0": 18, "true_0_pred_1": 3, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 0.9999574389413138, "unknown_mean_need_prob": 0.33543905473890756, "thr0.50": { "need_precision": 0.9553571428571429, "need_recall": 1.0, "need_f1": 0.9771689497716896, "specificity_unknown_refusal": 0.7619047619047619, "abstention_accuracy": 0.9609375, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.23809523809523808, "counts": { "tp": 107, "tn": 16, "fp": 5, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 1.907712697982788, "median_score_margin": 1.907712697982788, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9999270439147949, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 1.0, "recall1": 1.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 1.0, "ndcg3": 1.0, "mean_first_positive_rank": 1.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 4.908966302871704, "median_score_margin": 4.908966302871704, "negative_margin_rate": 0.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9988398551940918, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 7.163174152374268, "median_score_margin": 7.163174152374268, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9931761622428894, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.5, "recall5": 2.0, "recall8": 2.0, "mrr": 0.75, "ndcg3": 0.6532867981913646, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 0.5, "multi_positive_episodes": 2, "all_evidence_in_top3_multi": 0.5, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": 4.6119712591171265, "median_score_margin": 4.6119712591171265, "negative_margin_rate": 0.5, "hop_confusion": { "true_2_pred_1": 2, "true_2_pred_2": 0 }, "known_mean_need_prob": 0.998505711555481, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -1.0701351165771484, "median_score_margin": -1.0701351165771484, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9997996389865875, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.0, "recall8": 0.0, "mrr": 0.045, "ndcg3": 0.0, "mean_first_positive_rank": 22.5, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": -13.473207950592041, "median_score_margin": -13.473207950592041, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_0": 2, "true_1_pred_1": 0 }, "known_mean_need_prob": 0.027262956835329533, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": null, "need_recall": 0.0, "need_f1": null, "specificity_unknown_refusal": null, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": null, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2 } } }, "paraphrase": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 5.6224164962768555, "median_score_margin": 5.6224164962768555, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9998534619808197, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.5, "recall1": 0.5, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.75, "ndcg3": 0.8154648767857288, "mean_first_positive_rank": 1.5, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 5.72838020324707, "median_score_margin": 5.72838020324707, "negative_margin_rate": 0.5, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9995926022529602, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "mrr": 0.5, "ndcg3": 0.6309297535714575, "mean_first_positive_rank": 2.0, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.9848489761352539, "median_score_margin": -0.9848489761352539, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_1": 2 }, "known_mean_need_prob": 0.9999998807907104, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.6108048511576626, "recall1": 0.6108048511576626, "recall3": 0.9305402425578831, "recall5": 0.974090407938258, "recall8": 0.9939360529217199, "mrr": 0.7614842699193943, "ndcg3": 0.7944910797566598, "mean_first_positive_rank": 1.7844542447629548, "all_evidence_in_top3": 0.9305402425578831, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.996875, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 6.790419724446443, "median_score_margin": 2.042374363169074, "negative_margin_rate": 0.38919514884233736, "hop_confusion": { "true_0_pred_0": 100, "true_0_pred_1": 6, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9983032212039207, "unknown_mean_need_prob": 0.13497692132687178, "thr0.50": { "need_precision": 0.9945175438596491, "need_recall": 1.0, "need_f1": 0.9972512369433755, "specificity_unknown_refusal": 0.9056603773584906, "abstention_accuracy": 0.9947916666666666, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.09433962264150944, "counts": { "tp": 1814, "tn": 96, "fp": 10, "fn": 0 } } }, "unknown_abstention": { "episodes": 2, "answerable_episodes": 2, "unknown_episodes": 0, "top1": 0.0, "recall1": 0.0, "recall3": 0.0, "recall5": 0.0, "recall8": 0.0, "mrr": 0.03835227272727273, "ndcg3": 0.0, "mean_first_positive_rank": 27.0, "all_evidence_in_top3": 0.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.0, "hop_accuracy_answerable": 0.0, "hop_under_prediction": 1.0, "hop_over_prediction": 0.0, "mean_score_margin": -20.550126254558563, "median_score_margin": -20.550126254558563, "negative_margin_rate": 1.0, "hop_confusion": { "true_1_pred_0": 2, "true_1_pred_1": 0 }, "known_mean_need_prob": 0.03996259532868862, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": null, "need_recall": 0.0, "need_f1": null, "specificity_unknown_refusal": null, "abstention_accuracy": 0.0, "known_question_refusal_rate": 1.0, "unknown_question_read_rate": null, "counts": { "tp": 0, "tn": 0, "fp": 0, "fn": 2 } } } }, "protocol_consistency_delta": { "hop_accuracy": 0.0, "need_precision": 0.0, "need_recall": 0.0, "need_f1": 0.0, "abstention_accuracy": 0.0 } } }