WpyQwq
643e22ecb9
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线 - 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同） - 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00） - 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00% - 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益） - 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
3310 lines
104 KiB
JSON
3310 lines
104 KiB
JSON
{
|
|
"V2-512 best(step1000)": {
|
|
"checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_router_512\\router_best.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": 1000,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 4741902,
|
|
"checkpoint_bytes": 56922381
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048.0,
|
|
"checkpoint_bytes": 56922381
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 0.8951400030249109,
|
|
"single_query_latency_ms_p50": 0.8387999841943383,
|
|
"single_query_latency_ms_p95": 1.2322000111453235,
|
|
"queries_per_second": 1117.1436832459055,
|
|
"samples": 150
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.8213466650340706,
|
|
"single_query_latency_ms_p50": 0.8019999950192869,
|
|
"single_query_latency_ms_p95": 1.0353000252507627,
|
|
"queries_per_second": 1217.512705135923,
|
|
"samples": 150
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 1940,
|
|
"answerable_episodes": 1834,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.6712104689203926,
|
|
"recall1": 0.6712104689203926,
|
|
"recall3": 0.8876772082878953,
|
|
"recall5": 0.9492911668484187,
|
|
"recall8": 0.9858233369683751,
|
|
"mrr": 0.7908385671852008,
|
|
"ndcg3": 0.7999554748364778,
|
|
"mean_first_positive_rank": 1.841875681570338,
|
|
"all_evidence_in_top3": 0.8865866957470011,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.9783505154639175,
|
|
"hop_accuracy_answerable": 0.9967284623773174,
|
|
"hop_under_prediction": 0.003271537622682661,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.5880086265317487,
|
|
"median_score_margin": 0.3476818799972534,
|
|
"negative_margin_rate": 0.3287895310796074,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 70,
|
|
"true_0_pred_1": 36,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 4,
|
|
"true_1_pred_1": 1828,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.9847397770947952,
|
|
"unknown_mean_need_prob": 0.2536155528318629,
|
|
"thr0.30": {
|
|
"need_precision": 0.9786096256684492,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9881209503239742,
|
|
"specificity_unknown_refusal": 0.6226415094339622,
|
|
"abstention_accuracy": 0.977319587628866,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.37735849056603776,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 66,
|
|
"fp": 40,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.9796573875802997,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9886547811993517,
|
|
"specificity_unknown_refusal": 0.6415094339622641,
|
|
"abstention_accuracy": 0.9783505154639175,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.3584905660377358,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 68,
|
|
"fp": 38,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.9807073954983923,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9891891891891892,
|
|
"specificity_unknown_refusal": 0.660377358490566,
|
|
"abstention_accuracy": 0.979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.33962264150943394,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 70,
|
|
"fp": 36,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.9902597402597403,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9940249864204237,
|
|
"specificity_unknown_refusal": 0.8301886792452831,
|
|
"abstention_accuracy": 0.988659793814433,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.16981132075471697,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 88,
|
|
"fp": 18,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 0.9989082969432315,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9983633387888707,
|
|
"specificity_unknown_refusal": 0.9811320754716981,
|
|
"abstention_accuracy": 0.9969072164948454,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.018867924528301886,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 104,
|
|
"fp": 2,
|
|
"fn": 4
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.0,
|
|
"recall8": 0.0,
|
|
"mrr": 0.0,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 0.0,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.0,
|
|
"median_score_margin": 0.0,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": 0.0,
|
|
"unknown_mean_need_prob": 0.046122895131702535,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20,
|
|
"answerable_episodes": 20,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.4,
|
|
"recall1": 0.4,
|
|
"recall3": 0.8,
|
|
"recall5": 1.0,
|
|
"recall8": 1.05,
|
|
"mrr": 0.6225574712643678,
|
|
"ndcg3": 0.6086215664331643,
|
|
"mean_first_positive_rank": 3.45,
|
|
"all_evidence_in_top3": 0.7,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.7,
|
|
"hop_accuracy_answerable": 0.7,
|
|
"hop_under_prediction": 0.3,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.2963485062122345,
|
|
"median_score_margin": -0.25011467933654785,
|
|
"negative_margin_rate": 0.6,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 4,
|
|
"true_1_pred_1": 14,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.8251703456044197,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8,
|
|
"need_f1": 0.888888888888889,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8,
|
|
"known_question_refusal_rate": 0.2,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.6988869361452841,
|
|
"recall1": 0.6988869361452841,
|
|
"recall3": 0.9004100761570006,
|
|
"recall5": 0.9537199765670767,
|
|
"recall8": 0.9859402460456942,
|
|
"mrr": 0.8099543326670381,
|
|
"ndcg3": 0.8196676424617168,
|
|
"mean_first_positive_rank": 1.7492677211482133,
|
|
"all_evidence_in_top3": 0.9004100761570006,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 0.9895833333333334,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.6516403993389798,
|
|
"median_score_margin": 0.38489770889282227,
|
|
"negative_margin_rate": 0.30111306385471587,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 3,
|
|
"true_0_pred_1": 18,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9862654927744807,
|
|
"unknown_mean_need_prob": 0.5697948407559168,
|
|
"thr0.50": {
|
|
"need_precision": 0.9895652173913043,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9947552447552448,
|
|
"specificity_unknown_refusal": 0.14285714285714285,
|
|
"abstention_accuracy": 0.9895833333333334,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.8571428571428571,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 3,
|
|
"fp": 18,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.2803738317757009,
|
|
"recall1": 0.2803738317757009,
|
|
"recall3": 0.7009345794392523,
|
|
"recall5": 0.8691588785046729,
|
|
"recall8": 0.9719626168224299,
|
|
"mrr": 0.5173339899975414,
|
|
"ndcg3": 0.5212452695260414,
|
|
"mean_first_positive_rank": 3.0186915887850465,
|
|
"all_evidence_in_top3": 0.7009345794392523,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 0.859375,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.26182589241277393,
|
|
"median_score_margin": -0.27979564666748047,
|
|
"negative_margin_rate": 0.719626168224299,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 3,
|
|
"true_0_pred_1": 18,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 0.9902256833058651,
|
|
"unknown_mean_need_prob": 0.5697948407559168,
|
|
"thr0.50": {
|
|
"need_precision": 0.856,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9224137931034483,
|
|
"specificity_unknown_refusal": 0.14285714285714285,
|
|
"abstention_accuracy": 0.859375,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.8571428571428571,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 3,
|
|
"fp": 18,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.25075531005859375,
|
|
"median_score_margin": 0.25075531005859375,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9954056143760681,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 1.0,
|
|
"recall1": 1.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 1.0,
|
|
"ndcg3": 1.0,
|
|
"mean_first_positive_rank": 1.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 3.3609426021575928,
|
|
"median_score_margin": 3.3609426021575928,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.971575915813446,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 1.2087433338165283,
|
|
"median_score_margin": 1.2087433338165283,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.92432501912117,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.5,
|
|
"recall5": 2.0,
|
|
"recall8": 2.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.6934264036172708,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 10.865453004837036,
|
|
"median_score_margin": 10.865453004837036,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.9887627363204956,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 0.5,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.625,
|
|
"ndcg3": 0.5,
|
|
"mean_first_positive_rank": 2.5,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -1.6004958152770996,
|
|
"median_score_margin": -1.6004958152770996,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9807772934436798,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.5,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.375,
|
|
"ndcg3": 0.31546487678572877,
|
|
"mean_first_positive_rank": 3.0,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -3.42048117518425,
|
|
"median_score_margin": -3.42048117518425,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 2,
|
|
"true_1_pred_1": 0
|
|
},
|
|
"known_mean_need_prob": 0.1579926460981369,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.5,
|
|
"recall5": 0.5,
|
|
"recall8": 1.0,
|
|
"mrr": 0.3333333333333333,
|
|
"ndcg3": 0.31546487678572877,
|
|
"mean_first_positive_rank": 4.0,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -2.6525142192840576,
|
|
"median_score_margin": -2.6525142192840576,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.99253049492836,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 1.0,
|
|
"recall1": 1.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 1.0,
|
|
"ndcg3": 1.0,
|
|
"mean_first_positive_rank": 1.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 3.147550106048584,
|
|
"median_score_margin": 3.147550106048584,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9879558682441711,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.5,
|
|
"ndcg3": 0.6309297535714575,
|
|
"mean_first_positive_rank": 2.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.12020492553710938,
|
|
"median_score_margin": -0.12020492553710938,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9997169077396393,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.6742006615214995,
|
|
"recall1": 0.6742006615214995,
|
|
"recall3": 0.8886438809261301,
|
|
"recall5": 0.9487320837927232,
|
|
"recall8": 0.9851157662624035,
|
|
"mrr": 0.792693926566908,
|
|
"ndcg3": 0.8020649997361834,
|
|
"mean_first_positive_rank": 1.8241455347298787,
|
|
"all_evidence_in_top3": 0.8886438809261301,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 0.98125,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.5977589808067649,
|
|
"median_score_margin": 0.348813533782959,
|
|
"negative_margin_rate": 0.32579933847850057,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 70,
|
|
"true_0_pred_1": 36,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9864990872545568,
|
|
"unknown_mean_need_prob": 0.2536155528318629,
|
|
"thr0.50": {
|
|
"need_precision": 0.9805405405405405,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.990174672489083,
|
|
"specificity_unknown_refusal": 0.660377358490566,
|
|
"abstention_accuracy": 0.98125,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.33962264150943394,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 70,
|
|
"fp": 36,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.5,
|
|
"recall8": 0.5,
|
|
"mrr": 0.14224137931034483,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 16.5,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -14.003233283758163,
|
|
"median_score_margin": -14.003233283758163,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 2,
|
|
"true_1_pred_1": 0
|
|
},
|
|
"known_mean_need_prob": 0.25266095995903015,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"protocol_consistency_delta": {
|
|
"hop_accuracy": 0.0,
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"abstention_accuracy": 0.0
|
|
}
|
|
},
|
|
"V2-512 final(step100k)": {
|
|
"checkpoint": "H:\\Memory\\dynamic_memory_lab\\checkpoints\\natural_memory_v2_router_512\\memory_router_v2.pt",
|
|
"router": {
|
|
"kind": "router_v2",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_v2",
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3
|
|
},
|
|
"parameters": 4741902,
|
|
"checkpoint_bytes": 18973433
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048.0,
|
|
"checkpoint_bytes": 18973433
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 0.9346639970317483,
|
|
"single_query_latency_ms_p50": 0.8856999920681119,
|
|
"single_query_latency_ms_p95": 1.2600999907590449,
|
|
"queries_per_second": 1069.9031985566385,
|
|
"samples": 150
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 0.8593879973826309,
|
|
"single_query_latency_ms_p50": 0.8493000059388578,
|
|
"single_query_latency_ms_p95": 1.121999986935407,
|
|
"queries_per_second": 1163.6187648019518,
|
|
"samples": 150
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 1940,
|
|
"answerable_episodes": 1834,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.6063249727371864,
|
|
"recall1": 0.6063249727371864,
|
|
"recall3": 0.8500545256270448,
|
|
"recall5": 0.9013086150490731,
|
|
"recall8": 0.9487459105779716,
|
|
"mrr": 0.7325518172051582,
|
|
"ndcg3": 0.7456965349469862,
|
|
"mean_first_positive_rank": 2.6052344601962925,
|
|
"all_evidence_in_top3": 0.8489640130861504,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.9969072164948454,
|
|
"hop_accuracy_answerable": 0.9967284623773174,
|
|
"hop_under_prediction": 0.003271537622682661,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 3.8836881184794065,
|
|
"median_score_margin": 6.41892671585083,
|
|
"negative_margin_rate": 0.3936750272628135,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 4,
|
|
"true_1_pred_1": 1828,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.9977977412946947,
|
|
"unknown_mean_need_prob": 0.01519268750541802,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.0,
|
|
"recall8": 0.0,
|
|
"mrr": 0.0,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 0.0,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.0,
|
|
"median_score_margin": 0.0,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": 0.0,
|
|
"unknown_mean_need_prob": 8.623558289710172e-08,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20,
|
|
"answerable_episodes": 20,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.35,
|
|
"recall1": 0.35,
|
|
"recall3": 0.8,
|
|
"recall5": 0.9,
|
|
"recall8": 0.9,
|
|
"mrr": 0.5523787878787878,
|
|
"ndcg3": 0.5633897980311373,
|
|
"mean_first_positive_rank": 4.55,
|
|
"all_evidence_in_top3": 0.7,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.7,
|
|
"hop_accuracy_answerable": 0.7,
|
|
"hop_under_prediction": 0.3,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -12.90834836959839,
|
|
"median_score_margin": -9.948488235473633,
|
|
"negative_margin_rate": 0.65,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 4,
|
|
"true_1_pred_1": 14,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.8000006015426834,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8,
|
|
"need_f1": 0.888888888888889,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8,
|
|
"known_question_refusal_rate": 0.2,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.6256590509666081,
|
|
"recall1": 0.6256590509666081,
|
|
"recall3": 0.859402460456942,
|
|
"recall5": 0.9015817223198594,
|
|
"recall8": 0.9472759226713533,
|
|
"mrr": 0.7458644125569885,
|
|
"ndcg3": 0.7599420351849566,
|
|
"mean_first_positive_rank": 2.5606326889279436,
|
|
"all_evidence_in_top3": 0.859402460456942,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 5.392942665265922,
|
|
"median_score_margin": 7.422682553529739,
|
|
"negative_margin_rate": 0.37434094903339193,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.999977179556893,
|
|
"unknown_mean_need_prob": 0.038343318011833445,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.34579439252336447,
|
|
"recall1": 0.34579439252336447,
|
|
"recall3": 0.7102803738317757,
|
|
"recall5": 0.897196261682243,
|
|
"recall8": 0.9813084112149533,
|
|
"mrr": 0.5538495772140631,
|
|
"ndcg3": 0.552510234312422,
|
|
"mean_first_positive_rank": 2.953271028037383,
|
|
"all_evidence_in_top3": 0.7102803738317757,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -17.0551603077171,
|
|
"median_score_margin": -14.022430419921875,
|
|
"negative_margin_rate": 0.6542056074766355,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.038343318011833445,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.41666666666666663,
|
|
"ndcg3": 0.5654648767857288,
|
|
"mean_first_positive_rank": 2.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -10.757410049438477,
|
|
"median_score_margin": -10.757410049438477,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.6666666666666666,
|
|
"ndcg3": 0.75,
|
|
"mean_first_positive_rank": 2.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -12.85321569442749,
|
|
"median_score_margin": -12.85321569442749,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9999998807907104,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 11.889404773712158,
|
|
"median_score_margin": 11.889404773712158,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9999871253967285,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 1.0,
|
|
"recall1": 1.0,
|
|
"recall3": 1.5,
|
|
"recall5": 2.0,
|
|
"recall8": 2.0,
|
|
"mrr": 1.0,
|
|
"ndcg3": 0.8065735963827292,
|
|
"mean_first_positive_rank": 1.0,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 21.42271614074707,
|
|
"median_score_margin": 21.42271614074707,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.9999992251396179,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -24.237308502197266,
|
|
"median_score_margin": -24.237308502197266,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.0,
|
|
"recall8": 0.0,
|
|
"mrr": 0.0787878787878788,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 13.0,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -33.50318145751953,
|
|
"median_score_margin": -33.50318145751953,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 2,
|
|
"true_1_pred_1": 0
|
|
},
|
|
"known_mean_need_prob": 2.4256567030533915e-06,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.5,
|
|
"recall8": 0.5,
|
|
"mrr": 0.175,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 7.0,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -41.73876094818115,
|
|
"median_score_margin": -41.73876094818115,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 31.204745292663574,
|
|
"median_score_margin": 31.204745292663574,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.41666666666666663,
|
|
"ndcg3": 0.5654648767857288,
|
|
"mean_first_positive_rank": 2.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -12.233808517456055,
|
|
"median_score_margin": -12.233808517456055,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.6091510474090408,
|
|
"recall1": 0.6091510474090408,
|
|
"recall3": 0.850606394707828,
|
|
"recall5": 0.9013230429988974,
|
|
"recall8": 0.9492833517089305,
|
|
"mrr": 0.7345382894138281,
|
|
"ndcg3": 0.7477065320463893,
|
|
"mean_first_positive_rank": 2.5837927232635063,
|
|
"all_evidence_in_top3": 0.850606394707828,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 4.068826337752591,
|
|
"median_score_margin": 6.489462748169899,
|
|
"negative_margin_rate": 0.3908489525909592,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9999785256359517,
|
|
"unknown_mean_need_prob": 0.01519268750541802,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 0.5,
|
|
"recall5": 0.5,
|
|
"recall8": 0.5,
|
|
"mrr": 0.52,
|
|
"ndcg3": 0.5,
|
|
"mean_first_positive_rank": 13.0,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -58.27666473388672,
|
|
"median_score_margin": -58.27666473388672,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 2,
|
|
"true_1_pred_1": 0
|
|
},
|
|
"known_mean_need_prob": 1.735844307404477e-05,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"protocol_consistency_delta": {
|
|
"hop_accuracy": 0.0,
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"abstention_accuracy": 0.0
|
|
}
|
|
},
|
|
"XL-512 best": {
|
|
"checkpoint": "H:\\Memory\\V2_dpskw\\checkpoints\\router_xl_512\\router_best.pt",
|
|
"router": {
|
|
"kind": "router_xl",
|
|
"step": 3500,
|
|
"arch_config": {
|
|
"arch": "router_xl",
|
|
"arch_version": 1,
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3,
|
|
"encoder_layers": 2,
|
|
"encoder_hidden": 512,
|
|
"pair_blocks": 1,
|
|
"pair_hidden": 512,
|
|
"pair_expansion": 2,
|
|
"pair_dropout": 0.05,
|
|
"use_interaction": true,
|
|
"policy_layers": 2,
|
|
"policy_hidden": 512,
|
|
"policy_dropout": 0.0,
|
|
"learnable_cosine_scale": true
|
|
},
|
|
"parameters": 7898127,
|
|
"checkpoint_bytes": 99014505
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048.0,
|
|
"checkpoint_bytes": 99014505
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 1.3101640006061643,
|
|
"single_query_latency_ms_p50": 1.2462000013329089,
|
|
"single_query_latency_ms_p95": 1.7039999947883189,
|
|
"queries_per_second": 763.263224708767,
|
|
"samples": 150
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 1.719946669957911,
|
|
"single_query_latency_ms_p50": 1.7377999611198902,
|
|
"single_query_latency_ms_p95": 2.036199963185936,
|
|
"queries_per_second": 581.4133760464044,
|
|
"samples": 150
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 1940,
|
|
"answerable_episodes": 1834,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.608505997818975,
|
|
"recall1": 0.608505997818975,
|
|
"recall3": 0.9296619411123228,
|
|
"recall5": 0.9732824427480916,
|
|
"recall8": 0.9929116684841875,
|
|
"mrr": 0.759814160403073,
|
|
"ndcg3": 0.792764395065411,
|
|
"mean_first_positive_rank": 1.83206106870229,
|
|
"all_evidence_in_top3": 0.9285714285714286,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.9938144329896907,
|
|
"hop_accuracy_answerable": 0.9967284623773174,
|
|
"hop_under_prediction": 0.003271537622682661,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 6.709678303645345,
|
|
"median_score_margin": 2.013867676258087,
|
|
"negative_margin_rate": 0.3914940021810251,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 100,
|
|
"true_0_pred_1": 6,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 4,
|
|
"true_1_pred_1": 1828,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.9962027715833912,
|
|
"unknown_mean_need_prob": 0.13497691851203716,
|
|
"thr0.30": {
|
|
"need_precision": 0.9859913793103449,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9918699186991871,
|
|
"specificity_unknown_refusal": 0.7547169811320755,
|
|
"abstention_accuracy": 0.9845360824742269,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.24528301886792453,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 80,
|
|
"fp": 26,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 0.9924078091106291,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9951060358890701,
|
|
"specificity_unknown_refusal": 0.8679245283018868,
|
|
"abstention_accuracy": 0.9907216494845361,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.1320754716981132,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 92,
|
|
"fp": 14,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 0.9945652173913043,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9961894393032119,
|
|
"specificity_unknown_refusal": 0.9056603773584906,
|
|
"abstention_accuracy": 0.9927835051546392,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.09433962264150944,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 96,
|
|
"fp": 10,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 0.9989082969432315,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9983633387888707,
|
|
"specificity_unknown_refusal": 0.9811320754716981,
|
|
"abstention_accuracy": 0.9969072164948454,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.018867924528301886,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 104,
|
|
"fp": 2,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.0,
|
|
"recall8": 0.0,
|
|
"mrr": 0.0,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 0.0,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.0,
|
|
"median_score_margin": 0.0,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": 0.0,
|
|
"unknown_mean_need_prob": 0.003423646269766323,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20,
|
|
"answerable_episodes": 20,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.4,
|
|
"recall1": 0.4,
|
|
"recall3": 0.85,
|
|
"recall5": 0.9,
|
|
"recall8": 0.9,
|
|
"mrr": 0.6083352272727273,
|
|
"ndcg3": 0.6361540935691467,
|
|
"mean_first_positive_rank": 6.15,
|
|
"all_evidence_in_top3": 0.75,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.7,
|
|
"hop_accuracy_answerable": 0.7,
|
|
"hop_under_prediction": 0.3,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.6135687932372094,
|
|
"median_score_margin": -0.7090425491333008,
|
|
"negative_margin_rate": 0.6,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 4,
|
|
"true_1_pred_1": 14,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.8056919910013676,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8,
|
|
"need_f1": 0.888888888888889,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8,
|
|
"known_question_refusal_rate": 0.2,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.6239015817223199,
|
|
"recall1": 0.6239015817223199,
|
|
"recall3": 0.9420035149384886,
|
|
"recall5": 0.9806678383128296,
|
|
"recall8": 0.9953134153485648,
|
|
"mrr": 0.7716295930129063,
|
|
"ndcg3": 0.8064999615386277,
|
|
"mean_first_positive_rank": 1.7258347978910369,
|
|
"all_evidence_in_top3": 0.9420035149384886,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 0.9982638888888888,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 7.258645884769051,
|
|
"median_score_margin": 2.5352187156677246,
|
|
"negative_margin_rate": 0.37609841827768015,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 18,
|
|
"true_0_pred_1": 3,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9981995297581673,
|
|
"unknown_mean_need_prob": 0.33543904764311655,
|
|
"thr0.50": {
|
|
"need_precision": 0.9970794392523364,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9985375840889148,
|
|
"specificity_unknown_refusal": 0.7619047619047619,
|
|
"abstention_accuracy": 0.9971064814814815,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.23809523809523808,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 16,
|
|
"fp": 5,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.40186915887850466,
|
|
"recall1": 0.40186915887850466,
|
|
"recall3": 0.7476635514018691,
|
|
"recall5": 0.8691588785046729,
|
|
"recall8": 0.9719626168224299,
|
|
"mrr": 0.5996331809415921,
|
|
"ndcg3": 0.6029101339452666,
|
|
"mean_first_positive_rank": 2.7196261682242993,
|
|
"all_evidence_in_top3": 0.7476635514018691,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 0.9765625,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.6793190705650878,
|
|
"median_score_margin": -0.7896919250488281,
|
|
"negative_margin_rate": 0.5981308411214953,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 18,
|
|
"true_0_pred_1": 3,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 0.9999574389413138,
|
|
"unknown_mean_need_prob": 0.33543904764311655,
|
|
"thr0.50": {
|
|
"need_precision": 0.9553571428571429,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9771689497716896,
|
|
"specificity_unknown_refusal": 0.7619047619047619,
|
|
"abstention_accuracy": 0.9609375,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.23809523809523808,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 16,
|
|
"fp": 5,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 1.90769624710083,
|
|
"median_score_margin": 1.90769624710083,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9999270439147949,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 1.0,
|
|
"recall1": 1.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 1.0,
|
|
"ndcg3": 1.0,
|
|
"mean_first_positive_rank": 1.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 4.908981084823608,
|
|
"median_score_margin": 4.908981084823608,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9988398551940918,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 7.163186550140381,
|
|
"median_score_margin": 7.163186550140381,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9931761622428894,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.5,
|
|
"recall5": 2.0,
|
|
"recall8": 2.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.6532867981913646,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 4.611967086791992,
|
|
"median_score_margin": 4.611967086791992,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.998505711555481,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -1.0701360702514648,
|
|
"median_score_margin": -1.0701360702514648,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9997996389865875,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.0,
|
|
"recall8": 0.0,
|
|
"mrr": 0.045,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 22.5,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -13.473207950592041,
|
|
"median_score_margin": -13.473207950592041,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 2,
|
|
"true_1_pred_1": 0
|
|
},
|
|
"known_mean_need_prob": 0.02726295404136181,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 5.622410535812378,
|
|
"median_score_margin": 5.622410535812378,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9998534619808197,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 5.728376388549805,
|
|
"median_score_margin": 5.728376388549805,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9995926022529602,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.5,
|
|
"ndcg3": 0.6309297535714575,
|
|
"mean_first_positive_rank": 2.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -0.9848322868347168,
|
|
"median_score_margin": -0.9848322868347168,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9999998807907104,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.6108048511576626,
|
|
"recall1": 0.6108048511576626,
|
|
"recall3": 0.9305402425578831,
|
|
"recall5": 0.974090407938258,
|
|
"recall8": 0.9939360529217199,
|
|
"mrr": 0.7614842699193943,
|
|
"ndcg3": 0.7944910797566598,
|
|
"mean_first_positive_rank": 1.7844542447629548,
|
|
"all_evidence_in_top3": 0.9305402425578831,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 0.996875,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 6.790419726984734,
|
|
"median_score_margin": 2.0423730704933405,
|
|
"negative_margin_rate": 0.38919514884233736,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 100,
|
|
"true_0_pred_1": 6,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9983032212039207,
|
|
"unknown_mean_need_prob": 0.13497691851203716,
|
|
"thr0.50": {
|
|
"need_precision": 0.9945175438596491,
|
|
"need_recall": 1.0,
|
|
"need_f1": 0.9972512369433755,
|
|
"specificity_unknown_refusal": 0.9056603773584906,
|
|
"abstention_accuracy": 0.9947916666666666,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.09433962264150944,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 96,
|
|
"fp": 10,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.0,
|
|
"recall8": 0.0,
|
|
"mrr": 0.03835227272727273,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 27.0,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -20.550129517912865,
|
|
"median_score_margin": -20.550129517912865,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 2,
|
|
"true_1_pred_1": 0
|
|
},
|
|
"known_mean_need_prob": 0.03996259905397892,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"protocol_consistency_delta": {
|
|
"hop_accuracy": 0.0,
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"abstention_accuracy": 0.0
|
|
}
|
|
},
|
|
"XL-512 final(step100k)": {
|
|
"checkpoint": "H:\\Memory\\V2_dpskw\\checkpoints\\router_xl_512\\memory_router_xl.pt",
|
|
"router": {
|
|
"kind": "router_xl",
|
|
"step": null,
|
|
"arch_config": {
|
|
"arch": "router_xl",
|
|
"arch_version": 1,
|
|
"hidden_size": 2560,
|
|
"router_dim": 512,
|
|
"num_heads": 8,
|
|
"max_hops": 3,
|
|
"encoder_layers": 2,
|
|
"encoder_hidden": 512,
|
|
"pair_blocks": 1,
|
|
"pair_hidden": 512,
|
|
"pair_expansion": 2,
|
|
"pair_dropout": 0.05,
|
|
"use_interaction": true,
|
|
"policy_layers": 2,
|
|
"policy_hidden": 512,
|
|
"policy_dropout": 0.0,
|
|
"learnable_cosine_scale": true
|
|
},
|
|
"parameters": 7898127,
|
|
"checkpoint_bytes": 35802155
|
|
},
|
|
"storage": {
|
|
"address_bytes_per_record": 2048,
|
|
"address_mb_per_1m_records": 2048.0,
|
|
"checkpoint_bytes": 35802155
|
|
},
|
|
"latency": {
|
|
"cuda": {
|
|
"single_query_latency_ms_mean": 1.2458240000220637,
|
|
"single_query_latency_ms_p50": 1.1972999782301486,
|
|
"single_query_latency_ms_p95": 1.5769999590702355,
|
|
"queries_per_second": 802.6815986706708,
|
|
"samples": 150
|
|
},
|
|
"cpu": {
|
|
"single_query_latency_ms_mean": 1.7858959986673046,
|
|
"single_query_latency_ms_p50": 1.7789999837987125,
|
|
"single_query_latency_ms_p95": 2.090599969960749,
|
|
"queries_per_second": 559.9430206161128,
|
|
"samples": 150
|
|
}
|
|
},
|
|
"metrics": {
|
|
"episodes": 1940,
|
|
"answerable_episodes": 1834,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.5436205016357688,
|
|
"recall1": 0.5436205016357688,
|
|
"recall3": 0.8533260632497274,
|
|
"recall5": 0.9193020719738277,
|
|
"recall8": 0.9596510359869138,
|
|
"mrr": 0.6950444145765433,
|
|
"ndcg3": 0.7170573814087626,
|
|
"mean_first_positive_rank": 2.6161395856052345,
|
|
"all_evidence_in_top3": 0.8522355507088332,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.9969072164948454,
|
|
"hop_accuracy_answerable": 0.9967284623773174,
|
|
"hop_under_prediction": 0.003271537622682661,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 25.375655539715694,
|
|
"median_score_margin": 5.385736465454102,
|
|
"negative_margin_rate": 0.4563794983642312,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_0_pred_2": 0,
|
|
"true_1_pred_0": 4,
|
|
"true_1_pred_1": 1828,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.997793581256602,
|
|
"unknown_mean_need_prob": 0.012418151622086199,
|
|
"thr0.30": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.40": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.60": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
},
|
|
"thr0.70": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.9978189749182116,
|
|
"need_f1": 0.9989082969432315,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 0.9979381443298969,
|
|
"known_question_refusal_rate": 0.0021810250817884407,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1830,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
}
|
|
},
|
|
"by_family": {
|
|
"benchmark_qa": {
|
|
"episodes": 64,
|
|
"answerable_episodes": 0,
|
|
"unknown_episodes": 64,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.0,
|
|
"recall8": 0.0,
|
|
"mrr": 0.0,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 0.0,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 0.0,
|
|
"median_score_margin": 0.0,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 64
|
|
},
|
|
"known_mean_need_prob": 0.0,
|
|
"unknown_mean_need_prob": 8.598426478830835e-08,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 64,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"mega_validation": {
|
|
"episodes": 20,
|
|
"answerable_episodes": 20,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 0.8,
|
|
"recall5": 0.8,
|
|
"recall8": 0.85,
|
|
"mrr": 0.6361078431372549,
|
|
"ndcg3": 0.6455285910760186,
|
|
"mean_first_positive_rank": 6.8,
|
|
"all_evidence_in_top3": 0.7,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.7,
|
|
"hop_accuracy_answerable": 0.7,
|
|
"hop_under_prediction": 0.3,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 3.729068946838379,
|
|
"median_score_margin": 0.8376903533935547,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 4,
|
|
"true_1_pred_1": 14,
|
|
"true_1_pred_2": 0,
|
|
"true_2_pred_0": 0,
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.8000009355226269,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 0.8,
|
|
"need_f1": 0.888888888888889,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.8,
|
|
"known_question_refusal_rate": 0.2,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 16,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 4
|
|
}
|
|
}
|
|
},
|
|
"memory_policy": {
|
|
"episodes": 1728,
|
|
"answerable_episodes": 1707,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.5512595196250732,
|
|
"recall1": 0.5512595196250732,
|
|
"recall3": 0.8576449912126538,
|
|
"recall5": 0.9203280609256005,
|
|
"recall8": 0.9601640304628002,
|
|
"mrr": 0.7002585141169417,
|
|
"ndcg3": 0.7229373582956774,
|
|
"mean_first_positive_rank": 2.5735207967193907,
|
|
"all_evidence_in_top3": 0.8576449912126538,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 27.063399425814147,
|
|
"median_score_margin": 6.595245361328125,
|
|
"negative_margin_rate": 0.44874048037492675,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1707
|
|
},
|
|
"known_mean_need_prob": 0.9999727061008528,
|
|
"unknown_mean_need_prob": 0.03134091830829026,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1707,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"native_memory": {
|
|
"episodes": 128,
|
|
"answerable_episodes": 107,
|
|
"unknown_episodes": 21,
|
|
"top1": 0.42990654205607476,
|
|
"recall1": 0.42990654205607476,
|
|
"recall3": 0.794392523364486,
|
|
"recall5": 0.9252336448598131,
|
|
"recall8": 0.9719626168224299,
|
|
"mrr": 0.62287865301884,
|
|
"ndcg3": 0.6366223838451323,
|
|
"mean_first_positive_rank": 2.514018691588785,
|
|
"all_evidence_in_top3": 0.794392523364486,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 2.4967108508136784,
|
|
"median_score_margin": -7.090728759765625,
|
|
"negative_margin_rate": 0.5700934579439252,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 21,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 107
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.03134091830829026,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 107,
|
|
"tn": 21,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"by_category": {
|
|
"conflict_update": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 1.0507621765136719,
|
|
"median_score_margin": 1.0507621765136719,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_128": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 1.0,
|
|
"recall1": 1.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 1.0,
|
|
"ndcg3": 1.0,
|
|
"mean_first_positive_rank": 1.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 106.49523544311523,
|
|
"median_score_margin": 106.49523544311523,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"distractor_32": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 1.0,
|
|
"recall1": 1.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 1.0,
|
|
"ndcg3": 1.0,
|
|
"mean_first_positive_rank": 1.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 86.62086868286133,
|
|
"median_score_margin": 86.62086868286133,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 0.9999902844429016,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"forget_correction": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.5,
|
|
"recall5": 1.5,
|
|
"recall8": 2.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.6934264036172708,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 2,
|
|
"all_evidence_in_top3_multi": 0.5,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 53.472116470336914,
|
|
"median_score_margin": 53.472116470336914,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_2_pred_1": 2,
|
|
"true_2_pred_2": 0
|
|
},
|
|
"known_mean_need_prob": 0.9999991655349731,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"long_context": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.5,
|
|
"recall5": 0.5,
|
|
"recall8": 0.5,
|
|
"mrr": 0.27,
|
|
"ndcg3": 0.31546487678572877,
|
|
"mean_first_positive_rank": 13.5,
|
|
"all_evidence_in_top3": 0.5,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -53.50970458984375,
|
|
"median_score_margin": -53.50970458984375,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"multi_hop": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.0,
|
|
"recall8": 0.0,
|
|
"mrr": 0.04941176470588235,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 21.0,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -134.2181854248047,
|
|
"median_score_margin": -134.2181854248047,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 2,
|
|
"true_1_pred_1": 0
|
|
},
|
|
"known_mean_need_prob": 2.6863836524171347e-06,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2
|
|
}
|
|
}
|
|
},
|
|
"paraphrase": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 7.193164825439453,
|
|
"median_score_margin": 7.193164825439453,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"random_position": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 1.0,
|
|
"recall1": 1.0,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 1.0,
|
|
"ndcg3": 1.0,
|
|
"mean_first_positive_rank": 1.0,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 108.70876693725586,
|
|
"median_score_margin": 108.70876693725586,
|
|
"negative_margin_rate": 0.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"single_fact": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.5,
|
|
"recall1": 0.5,
|
|
"recall3": 1.0,
|
|
"recall5": 1.0,
|
|
"recall8": 1.0,
|
|
"mrr": 0.75,
|
|
"ndcg3": 0.8154648767857288,
|
|
"mean_first_positive_rank": 1.5,
|
|
"all_evidence_in_top3": 1.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -7.420085906982422,
|
|
"median_score_margin": -7.420085906982422,
|
|
"negative_margin_rate": 0.5,
|
|
"hop_confusion": {
|
|
"true_1_pred_1": 2
|
|
},
|
|
"known_mean_need_prob": 1.0,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 2,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown": {
|
|
"episodes": 1920,
|
|
"answerable_episodes": 1814,
|
|
"unknown_episodes": 106,
|
|
"top1": 0.5441014332965821,
|
|
"recall1": 0.5441014332965821,
|
|
"recall3": 0.8539140022050716,
|
|
"recall5": 0.9206174200661521,
|
|
"recall8": 0.9608599779492834,
|
|
"mrr": 0.695694211395058,
|
|
"ndcg3": 0.7178460119526738,
|
|
"mean_first_positive_rank": 2.570011025358324,
|
|
"all_evidence_in_top3": 0.8539140022050716,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 1.0,
|
|
"hop_accuracy_answerable": 1.0,
|
|
"hop_under_prediction": 0.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": 25.61431691339681,
|
|
"median_score_margin": 5.385736465454102,
|
|
"negative_margin_rate": 0.4558985667034179,
|
|
"hop_confusion": {
|
|
"true_0_pred_0": 106,
|
|
"true_0_pred_1": 0,
|
|
"true_1_pred_0": 0,
|
|
"true_1_pred_1": 1814
|
|
},
|
|
"known_mean_need_prob": 0.9999743160496999,
|
|
"unknown_mean_need_prob": 0.012418151622086199,
|
|
"thr0.50": {
|
|
"need_precision": 1.0,
|
|
"need_recall": 1.0,
|
|
"need_f1": 1.0,
|
|
"specificity_unknown_refusal": 1.0,
|
|
"abstention_accuracy": 1.0,
|
|
"known_question_refusal_rate": 0.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 1814,
|
|
"tn": 106,
|
|
"fp": 0,
|
|
"fn": 0
|
|
}
|
|
}
|
|
},
|
|
"unknown_abstention": {
|
|
"episodes": 2,
|
|
"answerable_episodes": 2,
|
|
"unknown_episodes": 0,
|
|
"top1": 0.0,
|
|
"recall1": 0.0,
|
|
"recall3": 0.0,
|
|
"recall5": 0.0,
|
|
"recall8": 0.0,
|
|
"mrr": 0.041666666666666664,
|
|
"ndcg3": 0.0,
|
|
"mean_first_positive_rank": 24.5,
|
|
"all_evidence_in_top3": 0.0,
|
|
"multi_positive_episodes": 0,
|
|
"all_evidence_in_top3_multi": 0.0,
|
|
"hop_accuracy": 0.0,
|
|
"hop_accuracy_answerable": 0.0,
|
|
"hop_under_prediction": 1.0,
|
|
"hop_over_prediction": 0.0,
|
|
"mean_score_margin": -131.1022491455078,
|
|
"median_score_margin": -131.1022491455078,
|
|
"negative_margin_rate": 1.0,
|
|
"hop_confusion": {
|
|
"true_1_pred_0": 2,
|
|
"true_1_pred_1": 0
|
|
},
|
|
"known_mean_need_prob": 1.7218864741153084e-05,
|
|
"unknown_mean_need_prob": 0.0,
|
|
"thr0.50": {
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"specificity_unknown_refusal": 0.0,
|
|
"abstention_accuracy": 0.0,
|
|
"known_question_refusal_rate": 1.0,
|
|
"unknown_question_read_rate": 0.0,
|
|
"counts": {
|
|
"tp": 0,
|
|
"tn": 0,
|
|
"fp": 0,
|
|
"fn": 2
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"protocol_consistency_delta": {
|
|
"hop_accuracy": 0.0,
|
|
"need_precision": 0.0,
|
|
"need_recall": 0.0,
|
|
"need_f1": 0.0,
|
|
"abstention_accuracy": 0.0
|
|
}
|
|
}
|
|
} |