{ "V2-128 v6 final": { "checkpoint": "checkpoints\\router_v6_v2_128\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 8156921 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 8156921 }, "latency": { "cuda": { "single_query_latency_ms_mean": 0.9690986667313458, "single_query_latency_ms_p50": 0.901999999769032, "single_query_latency_ms_p95": 1.4788999978918582, "queries_per_second": 1031.8866740090364, "samples": 150, "batched_qps_64": 77868.35385475321, "batched_latency_ms_64": 0.012842187493333768, "batched_qps_256": 275832.3454737125, "batched_latency_ms_256": 0.003625390627348679 }, "cpu": { "single_query_latency_ms_mean": 0.37916733340049785, "single_query_latency_ms_p50": 0.37180000072112307, "single_query_latency_ms_p95": 0.5995000028633513, "queries_per_second": 2637.3579997824963, "samples": 150, "batched_qps_64": 12773.737196355747, "batched_latency_ms_64": 0.07828562499980762, "batched_qps_256": 12928.000129629021, "batched_latency_ms_256": 0.07735148437291173 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9436959694622207, "recall1": 0.8882059054676097, "recall3": 0.9639609296059279, "recall5": 0.973981138430448, "recall8": 0.9864151790726395, "evidence_count_in_top3": 1.0717974626698104, "hit3": 0.9683956438756035, "mrr": 0.9597054333626427, "ndcg3": 0.9530820526370456, "mean_first_positive_rank": 1.2573818345121814, "all_evidence_in_top3": 0.9595262153362524, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.921, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 51.315328869764656, "median_score_margin": 56.234654903411865, "negative_margin_rate": 0.05630403053777928, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 15814, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999984274064, "unknown_mean_need_prob": 4.836221294362452e-06, "thr0.30": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 17814, "tn": 4106, "fp": 0, "fn": 0 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 4.8406160974828055e-06, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.9838125, "recall1": 0.92203125, "recall3": 0.9950625, "recall5": 0.99978125, "recall8": 0.999875, "evidence_count_in_top3": 1.115125, "hit3": 1.0, "mrr": 0.991875, "ndcg3": 0.9876817889701721, "mean_first_positive_rank": 1.016375, "all_evidence_in_top3": 0.990125, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.921, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 57.110110210259094, "median_score_margin": 59.90444374084473, "negative_margin_rate": 0.0161875, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 0, "true_1_pred_1": 14000, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999999701977, "unknown_mean_need_prob": 6.379481702447893e-11, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 16000, "tn": 4000, "fp": 0, "fn": 0 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.5940246045694201, "recall1": 0.5940246045694201, "recall3": 0.6842413591095489, "recall5": 0.7352079671939075, "recall8": 0.8611599297012302, "evidence_count_in_top3": 0.6842413591095489, "hit3": 0.6842413591095489, "mrr": 0.6760273585431921, "ndcg3": 0.6466497456649108, "mean_first_positive_rank": 3.4463971880492092, "all_evidence_in_top3": 0.6842413591095489, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.27154229221702947, "median_score_margin": 7.975786209106445, "negative_margin_rate": 0.40597539543058, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1707 }, "known_mean_need_prob": 0.9999999838679872, "unknown_mean_need_prob": 0.0004654159529796483, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1707, "tn": 21, "fp": 0, "fn": 0 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.5233644859813084, "recall1": 0.5233644859813084, "recall3": 0.7757009345794392, "recall5": 0.9252336448598131, "recall8": 0.9719626168224299, "evidence_count_in_top3": 0.7757009345794392, "hit3": 0.7757009345794392, "mrr": 0.674896157840083, "ndcg3": 0.6678873486315127, "mean_first_positive_rank": 2.3738317757009346, "all_evidence_in_top3": 0.7757009345794392, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -0.8786688875929217, "median_score_margin": 0.4756507873535156, "negative_margin_rate": 0.4766355140186916, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": 0.0004654159529796483, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.993, "recall1": 0.993, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9965, "ndcg3": 0.9974165082750002, "mean_first_positive_rank": 1.007, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 54.4210530708991, "median_score_margin": 56.92102670669556, "negative_margin_rate": 0.007, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.975, "recall1": 0.975, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9875, "ndcg3": 0.9907732438392864, "mean_first_positive_rank": 1.025, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 64.29146421498805, "median_score_margin": 73.40720510482788, "negative_margin_rate": 0.025, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9845, "recall1": 0.9845, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.99225, "ndcg3": 0.9942794111803576, "mean_first_positive_rank": 1.0155, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 65.52974343589041, "median_score_margin": 73.97303533554077, "negative_margin_rate": 0.0155, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 1.275862421441728e-10, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9795, "recall1": 0.9795, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9896666666666666, "ndcg3": 0.9923685950714292, "mean_first_positive_rank": 1.021, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 60.150181606799364, "median_score_margin": 68.01980018615723, "negative_margin_rate": 0.0205, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9885, "recall1": 0.49425, "recall3": 0.9605, "recall5": 0.99825, "recall8": 0.999, "evidence_count_in_top3": 1.921, "hit3": 1.0, "mrr": 0.99425, "ndcg3": 0.9452009954703022, "mean_first_positive_rank": 1.0115, "all_evidence_in_top3": 0.921, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.921, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 55.6439196386747, "median_score_margin": 62.50904202461243, "negative_margin_rate": 0.0115, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9999999997615814, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.995, "recall1": 0.995, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9975, "ndcg3": 0.9981546487678573, "mean_first_positive_rank": 1.005, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 48.13904330630228, "median_score_margin": 49.25835180282593, "negative_margin_rate": 0.005, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9805, "recall1": 0.9805, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9901666666666666, "ndcg3": 0.9927376653178577, "mean_first_positive_rank": 1.02, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 50.85701001610607, "median_score_margin": 54.9089093208313, "negative_margin_rate": 0.0195, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9745, "recall1": 0.9745, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9871666666666666, "ndcg3": 0.9905232438392865, "mean_first_positive_rank": 1.026, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 57.84846639241278, "median_score_margin": 63.093711376190186, "negative_margin_rate": 0.0255, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 1.0, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.5898566703417861, "recall1": 0.5898566703417861, "recall3": 0.6896361631753032, "recall5": 0.7464167585446527, "recall8": 0.8676957001102535, "evidence_count_in_top3": 0.6896361631753032, "hit3": 0.6896361631753032, "mrr": 0.6759606339151697, "ndcg3": 0.6479024598421029, "mean_first_positive_rank": 3.383131201764057, "all_evidence_in_top3": 0.6896361631753032, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 0.20369631854576997, "median_score_margin": 6.827788352966309, "negative_margin_rate": 0.4101433296582139, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 1814 }, "known_mean_need_prob": 0.9999999848195447, "unknown_mean_need_prob": 0.00018733273071117103, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 1814, "tn": 106, "fp": 0, "fn": 0 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 3.3919047850488803e-15, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } }, "ZOV-128 final": { "checkpoint": "checkpoints\\router_zov_v2_128\\memory_router_v2.pt", "router": { "kind": "router_v2", "step": null, "arch_config": { "arch": "router_v2", "hidden_size": 2560, "router_dim": 128, "num_heads": 8, "max_hops": 3 }, "parameters": 2037774, "checkpoint_bytes": 8156921 }, "storage": { "address_bytes_per_record": 512, "address_mb_per_1m_records": 512, "checkpoint_bytes": 8156921 }, "latency": { "cuda": { "single_query_latency_ms_mean": 1.0321086667924344, "single_query_latency_ms_p50": 0.9837000034167431, "single_query_latency_ms_p95": 1.4700000028824434, "queries_per_second": 968.8902265570339, "samples": 150, "batched_qps_64": 63339.997213904906, "batched_latency_ms_64": 0.015787812503731402, "batched_qps_256": 251226.69281453046, "batched_latency_ms_256": 0.00398046875034197 }, "cpu": { "single_query_latency_ms_mean": 0.395059999912822, "single_query_latency_ms_p50": 0.38700000004610047, "single_query_latency_ms_p95": 0.5204000044614077, "queries_per_second": 2531.2610748257744, "samples": 150, "batched_qps_64": 13292.67988538735, "batched_latency_ms_64": 0.07522937501107663, "batched_qps_256": 13652.896440499333, "batched_latency_ms_256": 0.07324453125079344 } }, "metrics": { "episodes": 21920, "answerable_episodes": 17814, "unknown_episodes": 4106, "top1": 0.9071516784551477, "recall1": 0.8525317166273717, "recall3": 0.9369035589985405, "recall5": 0.9511620074099023, "recall8": 0.9639047939822611, "evidence_count_in_top3": 1.0318288986190636, "hit3": 0.9542494667115752, "mrr": 0.9350004764720156, "ndcg3": 0.921200829570768, "mean_first_positive_rank": 1.555349724935444, "all_evidence_in_top3": 0.9195576512855058, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.691, "hop_accuracy": 0.9158759124087591, "hop_accuracy_answerable": 0.8964859099584597, "hop_under_prediction": 0.10351409004154036, "hop_over_prediction": 0.0, "mean_score_margin": 24.21305508213604, "median_score_margin": 25.759035110473633, "negative_margin_rate": 0.09284832154485237, "hop_confusion": { "true_0_pred_0": 4106, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 1844, "true_1_pred_1": 13970, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.868957594983749, "unknown_mean_need_prob": 1.5445844613853507e-10, "thr0.30": { "need_precision": 1.0, "need_recall": 0.9169754125968339, "need_f1": 0.956689800579812, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.9325273722627737, "known_question_refusal_rate": 0.08302458740316605, "unknown_question_read_rate": 0.0, "counts": { "tp": 16335, "tn": 4106, "fp": 0, "fn": 1479 } }, "thr0.40": { "need_precision": 1.0, "need_recall": 0.8759964073200853, "need_f1": 0.9338998773152997, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.8992244525547445, "known_question_refusal_rate": 0.12400359267991468, "unknown_question_read_rate": 0.0, "counts": { "tp": 15605, "tn": 4106, "fp": 0, "fn": 2209 } }, "thr0.50": { "need_precision": 1.0, "need_recall": 0.8615134164140563, "need_f1": 0.9256053798136364, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.8874543795620438, "known_question_refusal_rate": 0.13848658358594365, "unknown_question_read_rate": 0.0, "counts": { "tp": 15347, "tn": 4106, "fp": 0, "fn": 2467 } }, "thr0.60": { "need_precision": 1.0, "need_recall": 0.8427079824856855, "need_f1": 0.9146408334856516, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.8721715328467153, "known_question_refusal_rate": 0.1572920175143146, "unknown_question_read_rate": 0.0, "counts": { "tp": 15012, "tn": 4106, "fp": 0, "fn": 2802 } }, "thr0.70": { "need_precision": 1.0, "need_recall": 0.8322667564836645, "need_f1": 0.9084558823529412, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.8636861313868613, "known_question_refusal_rate": 0.16773324351633548, "unknown_question_read_rate": 0.0, "counts": { "tp": 14826, "tn": 4106, "fp": 0, "fn": 2988 } }, "thr0.80": { "need_precision": 1.0, "need_recall": 0.8274952284719883, "need_f1": 0.9056058977115651, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.859808394160584, "known_question_refusal_rate": 0.17250477152801166, "unknown_question_read_rate": 0.0, "counts": { "tp": 14741, "tn": 4106, "fp": 0, "fn": 3073 } } }, "by_family": { "benchmark_qa": { "episodes": 64, "answerable_episodes": 0, "unknown_episodes": 64, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 64 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 3.3175517012669046e-11, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 64, "fp": 0, "fn": 0 } } }, "mega_validation": { "episodes": 20000, "answerable_episodes": 16000, "unknown_episodes": 4000, "top1": 0.956125, "recall1": 0.8953125, "recall3": 0.9751875, "recall5": 0.9816875, "recall8": 0.98475, "evidence_count_in_top3": 1.080875, "hit3": 0.9945, "mrr": 0.9756561115666375, "ndcg3": 0.9636228207179964, "mean_first_positive_rank": 1.0985, "all_evidence_in_top3": 0.955875, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.691, "hop_accuracy": 0.92095, "hop_accuracy_answerable": 0.9011875, "hop_under_prediction": 0.0988125, "hop_over_prediction": 0.0, "mean_score_margin": 27.2492798316353, "median_score_margin": 28.251808643341064, "negative_margin_rate": 0.043875, "hop_confusion": { "true_0_pred_0": 4000, "true_0_pred_1": 0, "true_0_pred_2": 0, "true_1_pred_0": 1581, "true_1_pred_1": 12419, "true_1_pred_2": 0, "true_2_pred_0": 0, "true_2_pred_1": 0, "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.8787204854432493, "unknown_mean_need_prob": 3.7716760582211954e-16, "thr0.50": { "need_precision": 1.0, "need_recall": 0.8618125, "need_f1": 0.9257779717345328, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.88945, "known_question_refusal_rate": 0.1381875, "unknown_question_read_rate": 0.0, "counts": { "tp": 13789, "tn": 4000, "fp": 0, "fn": 2211 } } }, "memory_policy": { "episodes": 1728, "answerable_episodes": 1707, "unknown_episodes": 21, "top1": 0.4967779730521383, "recall1": 0.4967779730521383, "recall3": 0.6121851200937317, "recall5": 0.6877562975981254, "recall8": 0.7773872290568249, "evidence_count_in_top3": 0.6121851200937317, "hit3": 0.6121851200937317, "mrr": 0.5919114414798028, "ndcg3": 0.56399255386225, "mean_first_positive_rank": 5.587580550673697, "all_evidence_in_top3": 0.6121851200937317, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.8478009259259259, "hop_accuracy_answerable": 0.8459285295840656, "hop_under_prediction": 0.1540714704159344, "hop_over_prediction": 0.0, "mean_score_margin": -2.0129484753023483, "median_score_margin": -0.20529747009277344, "negative_margin_rate": 0.5032220269478618, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 263, "true_1_pred_1": 1444 }, "known_mean_need_prob": 0.7692814919056603, "unknown_mean_need_prob": 1.5049562811561687e-08, "thr0.50": { "need_precision": 1.0, "need_recall": 0.8500292911540714, "need_f1": 0.9189360354654844, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.8518518518518519, "known_question_refusal_rate": 0.14997070884592853, "unknown_question_read_rate": 0.0, "counts": { "tp": 1451, "tn": 21, "fp": 0, "fn": 256 } } }, "native_memory": { "episodes": 128, "answerable_episodes": 107, "unknown_episodes": 21, "top1": 0.1308411214953271, "recall1": 0.1308411214953271, "recall3": 0.3925233644859813, "recall5": 0.5887850467289719, "recall8": 0.822429906542056, "evidence_count_in_top3": 0.3925233644859813, "hit3": 0.3925233644859813, "mrr": 0.3290455345819019, "ndcg3": 0.2763659536715653, "mean_first_positive_rank": 5.542056074766355, "all_evidence_in_top3": 0.3925233644859813, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": -11.411317996750368, "median_score_margin": -10.944583892822266, "negative_margin_rate": 0.8691588785046729, "hop_confusion": { "true_0_pred_0": 21, "true_0_pred_1": 0, "true_1_pred_0": 0, "true_1_pred_1": 107 }, "known_mean_need_prob": 0.9992460118275936, "unknown_mean_need_prob": 1.5049562811561687e-08, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": 0.0, "counts": { "tp": 107, "tn": 21, "fp": 0, "fn": 0 } } } }, "by_category": { "conflict_update": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.968, "recall1": 0.968, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9838333333333333, "ndcg3": 0.9880588223607152, "mean_first_positive_rank": 1.033, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.509, "hop_accuracy_answerable": 0.509, "hop_under_prediction": 0.491, "hop_over_prediction": 0.0, "mean_score_margin": 26.64511842997372, "median_score_margin": 26.41234827041626, "negative_margin_rate": 0.032, "hop_confusion": { "true_1_pred_0": 982, "true_1_pred_1": 1018 }, "known_mean_need_prob": 0.3932232758104801, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 0.239, "need_f1": 0.3857949959644875, "specificity_unknown_refusal": null, "abstention_accuracy": 0.239, "known_question_refusal_rate": 0.761, "unknown_question_read_rate": null, "counts": { "tp": 478, "tn": 0, "fp": 0, "fn": 1522 } } }, "distractor_128": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.948, "recall1": 0.948, "recall3": 0.9995, "recall5": 0.9995, "recall8": 0.9995, "evidence_count_in_top3": 0.9995, "hit3": 0.9995, "mrr": 0.9736851851851852, "ndcg3": 0.9804274174321443, "mean_first_positive_rank": 1.065, "all_evidence_in_top3": 0.9995, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 25.91951792217791, "median_score_margin": 24.046737670898438, "negative_margin_rate": 0.052, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9997872362732887, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "distractor_32": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9575, "recall1": 0.9575, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9785833333333334, "ndcg3": 0.9841835847732154, "mean_first_positive_rank": 1.0435, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 25.73169531854242, "median_score_margin": 24.141990661621094, "negative_margin_rate": 0.0425, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9997921265959739, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "forget_correction": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 7.538130395219147e-16, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } }, "long_context": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.954, "recall1": 0.954, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9768333333333333, "ndcg3": 0.9828918389107156, "mean_first_positive_rank": 1.047, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 24.1671119158864, "median_score_margin": 20.684181213378906, "negative_margin_rate": 0.046, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9998738940358162, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "multi_hop": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.973, "recall1": 0.4865, "recall3": 0.8455, "recall5": 0.886, "recall8": 0.903, "evidence_count_in_top3": 1.691, "hit3": 1.0, "mrr": 0.9864166666666666, "ndcg3": 0.8584171108011072, "mean_first_positive_rank": 1.0275, "all_evidence_in_top3": 0.691, "multi_positive_episodes": 2000, "all_evidence_in_top3_multi": 0.691, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 31.524886593375356, "median_score_margin": 34.437350273132324, "negative_margin_rate": 0.027, "hop_confusion": { "true_2_pred_2": 2000 }, "known_mean_need_prob": 0.9990347371697426, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "paraphrase": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.982, "recall1": 0.982, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.9909166666666667, "ndcg3": 0.9932912706875006, "mean_first_positive_rank": 1.0185, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.8295, "hop_accuracy_answerable": 0.8295, "hop_under_prediction": 0.1705, "hop_over_prediction": 0.0, "mean_score_margin": 29.277884820342063, "median_score_margin": 29.79875612258911, "negative_margin_rate": 0.018, "hop_confusion": { "true_1_pred_0": 341, "true_1_pred_1": 1659 }, "known_mean_need_prob": 0.7633418458849192, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 0.7845, "need_f1": 0.8792378817595965, "specificity_unknown_refusal": null, "abstention_accuracy": 0.7845, "known_question_refusal_rate": 0.2155, "unknown_question_read_rate": null, "counts": { "tp": 1569, "tn": 0, "fp": 0, "fn": 431 } } }, "random_position": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.9035, "recall1": 0.9035, "recall3": 0.9565, "recall5": 0.968, "recall8": 0.9755, "evidence_count_in_top3": 0.9565, "hit3": 0.9565, "mrr": 0.9337303740145813, "ndcg3": 0.935564514526787, "mean_first_positive_rank": 1.515, "all_evidence_in_top3": 0.9565, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.871, "hop_accuracy_answerable": 0.871, "hop_under_prediction": 0.129, "hop_over_prediction": 0.0, "mean_score_margin": 22.111119223967194, "median_score_margin": 23.948859691619873, "negative_margin_rate": 0.0965, "hop_confusion": { "true_1_pred_0": 258, "true_1_pred_1": 1742 }, "known_mean_need_prob": 0.8747201321423054, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 0.871, "need_f1": 0.9310529128808124, "specificity_unknown_refusal": null, "abstention_accuracy": 0.871, "known_question_refusal_rate": 0.129, "unknown_question_read_rate": null, "counts": { "tp": 1742, "tn": 0, "fp": 0, "fn": 258 } } }, "single_fact": { "episodes": 2000, "answerable_episodes": 2000, "unknown_episodes": 0, "top1": 0.963, "recall1": 0.963, "recall3": 1.0, "recall5": 1.0, "recall8": 1.0, "evidence_count_in_top3": 1.0, "hit3": 1.0, "mrr": 0.98125, "ndcg3": 0.9861480062517868, "mean_first_positive_rank": 1.0385, "all_evidence_in_top3": 1.0, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": 1.0, "hop_under_prediction": 0.0, "hop_over_prediction": 0.0, "mean_score_margin": 32.616904428817335, "median_score_margin": 31.700976371765137, "negative_margin_rate": 0.037, "hop_confusion": { "true_1_pred_1": 2000 }, "known_mean_need_prob": 0.9999906356334687, "unknown_mean_need_prob": null, "thr0.50": { "need_precision": 1.0, "need_recall": 1.0, "need_f1": 1.0, "specificity_unknown_refusal": null, "abstention_accuracy": 1.0, "known_question_refusal_rate": 0.0, "unknown_question_read_rate": null, "counts": { "tp": 2000, "tn": 0, "fp": 0, "fn": 0 } } }, "unknown": { "episodes": 1920, "answerable_episodes": 1814, "unknown_episodes": 106, "top1": 0.47519294377067256, "recall1": 0.47519294377067256, "recall3": 0.5992282249173098, "recall5": 0.6819184123484013, "recall8": 0.7800441014332966, "evidence_count_in_top3": 0.5992282249173098, "hit3": 0.5992282249173098, "mrr": 0.5764061206208859, "ndcg3": 0.5470267069932294, "mean_first_positive_rank": 5.58489525909592, "all_evidence_in_top3": 0.5992282249173098, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 0.8630208333333333, "hop_accuracy_answerable": 0.8550165380374862, "hop_under_prediction": 0.14498346196251377, "hop_over_prediction": 0.0, "mean_score_margin": -2.5673175705586537, "median_score_margin": -1.374494194984436, "negative_margin_rate": 0.5248070562293274, "hop_confusion": { "true_0_pred_0": 106, "true_0_pred_1": 0, "true_1_pred_0": 263, "true_1_pred_1": 1551 }, "known_mean_need_prob": 0.7828461025074502, "unknown_mean_need_prob": 5.983064822400017e-09, "thr0.50": { "need_precision": 1.0, "need_recall": 0.8588754134509372, "need_f1": 0.9240806642941874, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 0.8666666666666667, "known_question_refusal_rate": 0.14112458654906285, "unknown_question_read_rate": 0.0, "counts": { "tp": 1558, "tn": 106, "fp": 0, "fn": 256 } } }, "unknown_abstention": { "episodes": 2000, "answerable_episodes": 0, "unknown_episodes": 2000, "top1": null, "recall1": null, "recall3": null, "recall5": null, "recall8": null, "evidence_count_in_top3": null, "hit3": null, "mrr": null, "ndcg3": null, "mean_first_positive_rank": null, "all_evidence_in_top3": null, "multi_positive_episodes": 0, "all_evidence_in_top3_multi": null, "hop_accuracy": 1.0, "hop_accuracy_answerable": null, "hop_under_prediction": null, "hop_over_prediction": null, "mean_score_margin": null, "median_score_margin": null, "negative_margin_rate": null, "hop_confusion": { "true_0_pred_0": 2000 }, "known_mean_need_prob": null, "unknown_mean_need_prob": 5.221721223244015e-19, "thr0.50": { "need_precision": null, "need_recall": null, "need_f1": null, "specificity_unknown_refusal": 1.0, "abstention_accuracy": 1.0, "known_question_refusal_rate": null, "unknown_question_read_rate": 0.0, "counts": { "tp": 0, "tn": 2000, "fp": 0, "fn": 0 } } } } } }