WpyQwq
643e22ecb9
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线 - 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同） - 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00） - 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00% - 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益） - 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
812 lines
26 KiB
JSON
812 lines
26 KiB
JSON
{
|
|
"V2-128 v6 best": {
|
|
"checkpoint": "checkpoints\\router_v6_v2_128\\router_best.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": 6000,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 2037774,
|
|
"checkpoint_bytes": 24472973
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 24472973
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 0.8666539997405683,
|
|
"single_query_latency_ms_p50": 0.844600006530527,
|
|
"single_query_latency_ms_p95": 1.1691999970935285,
|
|
"queries_per_second": 1153.8630183433627,
|
|
"samples": 150,
|
|
"batched_qps_64": 61288.5926494177,
|
|
"batched_latency_ms_64": 0.016316250003001187,
|
|
"batched_qps_256": 273229.8761194092,
|
|
"batched_latency_ms_256": 0.0036599218731225847
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.33684533375587006,
|
|
"single_query_latency_ms_p50": 0.3316000002087094,
|
|
"single_query_latency_ms_p95": 0.46629999997094274,
|
|
"queries_per_second": 2968.721546027869,
|
|
"samples": 150,
|
|
"batched_qps_64": 15424.808876649771,
|
|
"batched_latency_ms_64": 0.06483062500137748,
|
|
"batched_qps_256": 17376.784343192005,
|
|
"batched_latency_ms_256": 0.057548046879674075
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 300,
|
|
"answerable_episodes": 250,
|
|
"unknown_episodes": 50,
|
|
"top1": 0.076,
|
|
"recall1": 0.076,
|
|
"recall3": 0.236,
|
|
"recall5": 0.372,
|
|
"recall8": 0.488,
|
|
"evidence_count_in_top3": 0.236,
|
|
"hit3": 0.236,
|
|
"mrr": 0.22881929448111302,
|
|
"ndcg3": 0.1664743802857166,
|
|
"mean_first_positive_rank": 9.448,
|
|
"all_evidence_in_top3": 0.236,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.8333333333333334,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -7.889600224852562,
|
|
"median_score_margin": -7.62301778793335,
|
|
"negative_margin_rate": 0.924,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 50,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 250
|
|
},
|
|
"known_mean_need_prob": 0.9972750458717347,
|
|
"unknown_mean_need_prob": 0.9953130459785462,
|
|
"thr0.30": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"zero_overlap_paraphrase": {
|
|
"episodes": 300,
|
|
"answerable_episodes": 250,
|
|
"unknown_episodes": 50,
|
|
"top1": 0.076,
|
|
"recall1": 0.076,
|
|
"recall3": 0.236,
|
|
"recall5": 0.372,
|
|
"recall8": 0.488,
|
|
"evidence_count_in_top3": 0.236,
|
|
"hit3": 0.236,
|
|
"mrr": 0.22881929448111302,
|
|
"ndcg3": 0.1664743802857166,
|
|
"mean_first_positive_rank": 9.448,
|
|
"all_evidence_in_top3": 0.236,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.8333333333333334,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -7.889600224852562,
|
|
"median_score_margin": -7.62301778793335,
|
|
"negative_margin_rate": 0.924,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 50,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 250
|
|
},
|
|
"known_mean_need_prob": 0.9972750458717347,
|
|
"unknown_mean_need_prob": 0.9953130459785462,
|
|
"thr0.50": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"zero_overlap_paraphrase": {
|
|
"episodes": 300,
|
|
"answerable_episodes": 250,
|
|
"unknown_episodes": 50,
|
|
"top1": 0.076,
|
|
"recall1": 0.076,
|
|
"recall3": 0.236,
|
|
"recall5": 0.372,
|
|
"recall8": 0.488,
|
|
"evidence_count_in_top3": 0.236,
|
|
"hit3": 0.236,
|
|
"mrr": 0.22881929448111302,
|
|
"ndcg3": 0.1664743802857166,
|
|
"mean_first_positive_rank": 9.448,
|
|
"all_evidence_in_top3": 0.236,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.8333333333333334,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -7.889600224852562,
|
|
"median_score_margin": -7.62301778793335,
|
|
"negative_margin_rate": 0.924,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 50,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 250
|
|
},
|
|
"known_mean_need_prob": 0.9972750458717347,
|
|
"unknown_mean_need_prob": 0.9953130459785462,
|
|
"thr0.50": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"V2-128 v6 final": {
|
|
"checkpoint": "checkpoints\\router_v6_v2_128\\memory_router_v2.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 2037774,
|
|
"checkpoint_bytes": 8156921
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 8156921
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 0.8889626660675276,
|
|
"single_query_latency_ms_p50": 0.8526999954483472,
|
|
"single_query_latency_ms_p95": 1.1702999981935136,
|
|
"queries_per_second": 1124.9066335076413,
|
|
"samples": 150,
|
|
"batched_qps_64": 75940.95592518139,
|
|
"batched_latency_ms_64": 0.013168124996809638,
|
|
"batched_qps_256": 296056.43592556706,
|
|
"batched_latency_ms_256": 0.0033777343730889697
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.3601646669267211,
|
|
"single_query_latency_ms_p50": 0.35260000004200265,
|
|
"single_query_latency_ms_p95": 0.5258000019239262,
|
|
"queries_per_second": 2776.5077805465558,
|
|
"samples": 150,
|
|
"batched_qps_64": 14627.50153387338,
|
|
"batched_latency_ms_64": 0.06836437498805026,
|
|
"batched_qps_256": 17832.662752528264,
|
|
"batched_latency_ms_256": 0.05607687499491476
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 300,
|
|
"answerable_episodes": 250,
|
|
"unknown_episodes": 50,
|
|
"top1": 0.184,
|
|
"recall1": 0.184,
|
|
"recall3": 0.36,
|
|
"recall5": 0.604,
|
|
"recall8": 0.708,
|
|
"evidence_count_in_top3": 0.36,
|
|
"hit3": 0.36,
|
|
"mrr": 0.3507031323442415,
|
|
"ndcg3": 0.28404553732857407,
|
|
"mean_first_positive_rank": 6.792,
|
|
"all_evidence_in_top3": 0.36,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.8333333333333334,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -18.688180733680724,
|
|
"median_score_margin": -16.592705011367798,
|
|
"negative_margin_rate": 0.816,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 50,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 250
|
|
},
|
|
"known_mean_need_prob": 0.9918208360671997,
|
|
"unknown_mean_need_prob": 0.9822168827056885,
|
|
"thr0.30": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"zero_overlap_paraphrase": {
|
|
"episodes": 300,
|
|
"answerable_episodes": 250,
|
|
"unknown_episodes": 50,
|
|
"top1": 0.184,
|
|
"recall1": 0.184,
|
|
"recall3": 0.36,
|
|
"recall5": 0.604,
|
|
"recall8": 0.708,
|
|
"evidence_count_in_top3": 0.36,
|
|
"hit3": 0.36,
|
|
"mrr": 0.3507031323442415,
|
|
"ndcg3": 0.28404553732857407,
|
|
"mean_first_positive_rank": 6.792,
|
|
"all_evidence_in_top3": 0.36,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.8333333333333334,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -18.688180733680724,
|
|
"median_score_margin": -16.592705011367798,
|
|
"negative_margin_rate": 0.816,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 50,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 250
|
|
},
|
|
"known_mean_need_prob": 0.9918208360671997,
|
|
"unknown_mean_need_prob": 0.9822168827056885,
|
|
"thr0.50": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"zero_overlap_paraphrase": {
|
|
"episodes": 300,
|
|
"answerable_episodes": 250,
|
|
"unknown_episodes": 50,
|
|
"top1": 0.184,
|
|
"recall1": 0.184,
|
|
"recall3": 0.36,
|
|
"recall5": 0.604,
|
|
"recall8": 0.708,
|
|
"evidence_count_in_top3": 0.36,
|
|
"hit3": 0.36,
|
|
"mrr": 0.3507031323442415,
|
|
"ndcg3": 0.28404553732857407,
|
|
"mean_first_positive_rank": 6.792,
|
|
"all_evidence_in_top3": 0.36,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.8333333333333334,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -18.688180733680724,
|
|
"median_score_margin": -16.592705011367798,
|
|
"negative_margin_rate": 0.816,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 0,
|
|
"true_0_pred_1": 50,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 250
|
|
},
|
|
"known_mean_need_prob": 0.9918208360671997,
|
|
"unknown_mean_need_prob": 0.9822168827056885,
|
|
"thr0.50": {
|
|
"need_precision": 0.8333333333333334,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9090909090909091,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8333333333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 1.0,
|
|
"counts": {
|
|
"tp": 250,
|
|
"tn": 0,
|
|
"fp": 50,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"V2-128 deployed(v3)": {
|
|
"checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_qwen_router_entities\\memory_router_v2.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 128,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 2037774,
|
|
"checkpoint_bytes": 8157561
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 512,
|
|
"address_mb_per_1m_records": 512,
|
|
"checkpoint_bytes": 8157561
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 0.9203346665890422,
|
|
"single_query_latency_ms_p50": 0.9079000010387972,
|
|
"single_query_latency_ms_p95": 1.215000003867317,
|
|
"queries_per_second": 1086.561265486407,
|
|
"samples": 150,
|
|
"batched_qps_64": 71565.95256269597,
|
|
"batched_latency_ms_64": 0.013973124987387564,
|
|
"batched_qps_256": 257898.13019024945,
|
|
"batched_latency_ms_256": 0.003877500000726286
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.36491733257814,
|
|
"single_query_latency_ms_p50": 0.338099998771213,
|
|
"single_query_latency_ms_p95": 0.5211000025155954,
|
|
"queries_per_second": 2740.3466777941253,
|
|
"samples": 150,
|
|
"batched_qps_64": 15065.77151150895,
|
|
"batched_latency_ms_64": 0.06637562498781335,
|
|
"batched_qps_256": 15734.809454760149,
|
|
"batched_latency_ms_256": 0.06355335937655582
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 300,
|
|
"answerable_episodes": 250,
|
|
"unknown_episodes": 50,
|
|
"top1": 0.116,
|
|
"recall1": 0.116,
|
|
"recall3": 0.32,
|
|
"recall5": 0.52,
|
|
"recall8": 0.64,
|
|
"evidence_count_in_top3": 0.32,
|
|
"hit3": 0.32,
|
|
"mrr": 0.2947678317950427,
|
|
"ndcg3": 0.23475900845714656,
|
|
"mean_first_positive_rank": 7.472,
|
|
"all_evidence_in_top3": 0.32,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.6233333333333333,
|
|
"hop_accuracy_answerable": 0.708,
|
|
"hop_under_prediction": 0.228,
|
|
"hop_over_prediction": 0.064,
|
|
"mean_score_margin": -22.876055246174335,
|
|
"median_score_margin": -17.429455399513245,
|
|
"negative_margin_rate": 0.884,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 10,
|
|
"true_0_pred_1": 39,
|
|
"true_0_pred_2": 1,
|
|
"true_1_pred_0": 57,
|
|
"true_1_pred_1": 177,
|
|
"true_1_pred_2": 16
|
|
},
|
|
"known_mean_need_prob": 0.7523725980222226,
|
|
"unknown_mean_need_prob": 0.8253701922297477,
|
|
"thr0.30": {
|
|
"need_precision": 0.8185328185328186,
|
|
"need_recall": 0.848,
|
|
"need_f1": 0.8330058939096268,
|
|
"specificity_unknown_refusal": 0.06,
|
|
"abstention_accuracy": 0.7166666666666667,
|
|
"known_question_refusal_rate": 0.152,
|
|
"unknown_question_read_rate": 0.94,
|
|
"counts": {
|
|
"tp": 212,
|
|
"tn": 3,
|
|
"fp": 47,
|
|
"fn": 38
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.8185328185328186,
|
|
"need_recall": 0.848,
|
|
"need_f1": 0.8330058939096268,
|
|
"specificity_unknown_refusal": 0.06,
|
|
"abstention_accuracy": 0.7166666666666667,
|
|
"known_question_refusal_rate": 0.152,
|
|
"unknown_question_read_rate": 0.94,
|
|
"counts": {
|
|
"tp": 212,
|
|
"tn": 3,
|
|
"fp": 47,
|
|
"fn": 38
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.8185328185328186,
|
|
"need_recall": 0.848,
|
|
"need_f1": 0.8330058939096268,
|
|
"specificity_unknown_refusal": 0.06,
|
|
"abstention_accuracy": 0.7166666666666667,
|
|
"known_question_refusal_rate": 0.152,
|
|
"unknown_question_read_rate": 0.94,
|
|
"counts": {
|
|
"tp": 212,
|
|
"tn": 3,
|
|
"fp": 47,
|
|
"fn": 38
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.8185328185328186,
|
|
"need_recall": 0.848,
|
|
"need_f1": 0.8330058939096268,
|
|
"specificity_unknown_refusal": 0.06,
|
|
"abstention_accuracy": 0.7166666666666667,
|
|
"known_question_refusal_rate": 0.152,
|
|
"unknown_question_read_rate": 0.94,
|
|
"counts": {
|
|
"tp": 212,
|
|
"tn": 3,
|
|
"fp": 47,
|
|
"fn": 38
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 0.8025751072961373,
|
|
"need_recall": 0.748,
|
|
"need_f1": 0.7743271221532091,
|
|
"specificity_unknown_refusal": 0.08,
|
|
"abstention_accuracy": 0.6366666666666667,
|
|
"known_question_refusal_rate": 0.252,
|
|
"unknown_question_read_rate": 0.92,
|
|
"counts": {
|
|
"tp": 187,
|
|
"tn": 4,
|
|
"fp": 46,
|
|
"fn": 63
|
|
}
|
|
},
|
|
"thr0.80": {
|
|
"need_precision": 0.815028901734104,
|
|
"need_recall": 0.564,
|
|
"need_f1": 0.6666666666666666,
|
|
"specificity_unknown_refusal": 0.36,
|
|
"abstention_accuracy": 0.53,
|
|
"known_question_refusal_rate": 0.436,
|
|
"unknown_question_read_rate": 0.64,
|
|
"counts": {
|
|
"tp": 141,
|
|
"tn": 18,
|
|
"fp": 32,
|
|
"fn": 109
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"zero_overlap_paraphrase": {
|
|
"episodes": 300,
|
|
"answerable_episodes": 250,
|
|
"unknown_episodes": 50,
|
|
"top1": 0.116,
|
|
"recall1": 0.116,
|
|
"recall3": 0.32,
|
|
"recall5": 0.52,
|
|
"recall8": 0.64,
|
|
"evidence_count_in_top3": 0.32,
|
|
"hit3": 0.32,
|
|
"mrr": 0.2947678317950427,
|
|
"ndcg3": 0.23475900845714656,
|
|
"mean_first_positive_rank": 7.472,
|
|
"all_evidence_in_top3": 0.32,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.6233333333333333,
|
|
"hop_accuracy_answerable": 0.708,
|
|
"hop_under_prediction": 0.228,
|
|
"hop_over_prediction": 0.064,
|
|
"mean_score_margin": -22.876055246174335,
|
|
"median_score_margin": -17.429455399513245,
|
|
"negative_margin_rate": 0.884,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 10,
|
|
"true_0_pred_1": 39,
|
|
"true_0_pred_2": 1,
|
|
"true_1_pred_0": 57,
|
|
"true_1_pred_1": 177,
|
|
"true_1_pred_2": 16
|
|
},
|
|
"known_mean_need_prob": 0.7523725980222226,
|
|
"unknown_mean_need_prob": 0.8253701922297477,
|
|
"thr0.50": {
|
|
"need_precision": 0.8185328185328186,
|
|
"need_recall": 0.848,
|
|
"need_f1": 0.8330058939096268,
|
|
"specificity_unknown_refusal": 0.06,
|
|
"abstention_accuracy": 0.7166666666666667,
|
|
"known_question_refusal_rate": 0.152,
|
|
"unknown_question_read_rate": 0.94,
|
|
"counts": {
|
|
"tp": 212,
|
|
"tn": 3,
|
|
"fp": 47,
|
|
"fn": 38
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"zero_overlap_paraphrase": {
|
|
"episodes": 300,
|
|
"answerable_episodes": 250,
|
|
"unknown_episodes": 50,
|
|
"top1": 0.116,
|
|
"recall1": 0.116,
|
|
"recall3": 0.32,
|
|
"recall5": 0.52,
|
|
"recall8": 0.64,
|
|
"evidence_count_in_top3": 0.32,
|
|
"hit3": 0.32,
|
|
"mrr": 0.2947678317950427,
|
|
"ndcg3": 0.23475900845714656,
|
|
"mean_first_positive_rank": 7.472,
|
|
"all_evidence_in_top3": 0.32,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": null,
|
|
"hop_accuracy": 0.6233333333333333,
|
|
"hop_accuracy_answerable": 0.708,
|
|
"hop_under_prediction": 0.228,
|
|
"hop_over_prediction": 0.064,
|
|
"mean_score_margin": -22.876055246174335,
|
|
"median_score_margin": -17.429455399513245,
|
|
"negative_margin_rate": 0.884,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 10,
|
|
"true_0_pred_1": 39,
|
|
"true_0_pred_2": 1,
|
|
"true_1_pred_0": 57,
|
|
"true_1_pred_1": 177,
|
|
"true_1_pred_2": 16
|
|
},
|
|
"known_mean_need_prob": 0.7523725980222226,
|
|
"unknown_mean_need_prob": 0.8253701922297477,
|
|
"thr0.50": {
|
|
"need_precision": 0.8185328185328186,
|
|
"need_recall": 0.848,
|
|
"need_f1": 0.8330058939096268,
|
|
"specificity_unknown_refusal": 0.06,
|
|
"abstention_accuracy": 0.7166666666666667,
|
|
"known_question_refusal_rate": 0.152,
|
|
"unknown_question_read_rate": 0.94,
|
|
"counts": {
|
|
"tp": 212,
|
|
"tn": 3,
|
|
"fp": 47,
|
|
"fn": 38
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
} |