wpy/natural-memory-nm21
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
643e22ecb905392eedef8b52ce7c9d0c7f185ed1
natural-memory-nm21/nm2_1_fix_A_e2e.json
T
WpyQwq 643e22ecb9 Natural Memory NM2.1: 记忆路由器分叉、数据集缺陷修复与全轴评测证据 
- 引入 MemoryRouterXL 与 v5/v6 流式多线程训练/编码管线
- 修复 prepare_memory_router_dataset 候选池重建缺陷（mega 家族 3568x 加速，输出逐字节相同）
- 修复 v5 被破坏的拒答与多跳标签（train 未知样本 319 -> 16319，multi_hop 平均正例 1.00 -> 2.00）
- 同存储预算下 V2-128 v6 逐轴 22/22 通过：Top-1 41.12% -> 94.62%，未知拒答 0.00% -> 100.00%
- 记录三条被实测推翻的显然优化（logits_to_keep=1 反而慢 55%、XL 容量未带来收益）
- 记忆手术跨架构可移植性 14/14，读写关闭时与原生模型逐位相同
2026-09-19 11:11:31 +08:00

1367 lines
47 KiB
JSON
Raw Blame History

{
"NM2.1 合体": {
"summary": {
"router": "NM2.1 合体",
"cases": 110,
"accuracy_pct": 88.18181818181819,
"answerable_cases": 80,
"answerable_accuracy_pct": 100.0,
"unknown_cases": 30,
"unknown_refusal_pct": 56.666666666666664,
"wrong_abstention_pct": 0.0,
"seconds": 301.5,
"per_category": {
"(policy)": {
"cases": 10,
"accuracy_pct": 0.0,
"answerable": 0,
"answerable_accuracy_pct": 0.0,
"unknown": 10,
"unknown_refusal_pct": 0.0
},
"conflict_update": {
"cases": 10,
"accuracy_pct": 100.0,
"answerable": 10,
"answerable_accuracy_pct": 100.0,
"unknown": 0,
"unknown_refusal_pct": 0.0
},
"distractor_128": {
"cases": 10,
"accuracy_pct": 100.0,
"answerable": 10,
"answerable_accuracy_pct": 100.0,
"unknown": 0,
"unknown_refusal_pct": 0.0
},
"distractor_32": {
"cases": 10,
"accuracy_pct": 100.0,
"answerable": 10,
"answerable_accuracy_pct": 100.0,
"unknown": 0,
"unknown_refusal_pct": 0.0
},
"forget_correction": {
"cases": 10,
"accuracy_pct": 70.0,
"answerable": 0,
"answerable_accuracy_pct": 0.0,
"unknown": 10,
"unknown_refusal_pct": 70.0
},
"long_context": {
"cases": 10,
"accuracy_pct": 100.0,
"answerable": 10,
"answerable_accuracy_pct": 100.0,
"unknown": 0,
"unknown_refusal_pct": 0.0
},
"multi_hop": {
"cases": 10,
"accuracy_pct": 100.0,
"answerable": 10,
"answerable_accuracy_pct": 100.0,
"unknown": 0,
"unknown_refusal_pct": 0.0
},
"paraphrase": {
"cases": 10,
"accuracy_pct": 100.0,
"answerable": 10,
"answerable_accuracy_pct": 100.0,
"unknown": 0,
"unknown_refusal_pct": 0.0
},
"random_position": {
"cases": 10,
"accuracy_pct": 100.0,
"answerable": 10,
"answerable_accuracy_pct": 100.0,
"unknown": 0,
"unknown_refusal_pct": 0.0
},
"single_fact": {
"cases": 10,
"accuracy_pct": 100.0,
"answerable": 10,
"answerable_accuracy_pct": 100.0,
"unknown": 0,
"unknown_refusal_pct": 0.0
},
"unknown_abstention": {
"cases": 10,
"accuracy_pct": 100.0,
"answerable": 0,
"answerable_accuracy_pct": 0.0,
"unknown": 10,
"unknown_refusal_pct": 100.0
}
}
},
"rows": [
{
"category": "(policy)",
"query": "请问对象测试实体0000的驻地代号为？",
"reply": "关于对象测试实体0000的驻地代号，当前长期记忆中无相关记录，无法提供确切答案。",
"written": 6,
"correct": false,
"matched": [],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "(policy)",
"query": "查询：测试实体0001的负责人代号是什么？",
"reply": "测试实体 0001 的负责人代号是成员-M-SBOIIYKN。",
"written": 6,
"correct": false,
"matched": [],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "(policy)",
"query": "查询：测试实体0002的维护日代号是什么？",
"reply": "关于测试实体0002的维护日代号，当前长期记忆中无相关记录，无法回答。",
"written": 6,
"correct": false,
"matched": [],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "(policy)",
"query": "请问对象测试实体0003的安全级别代号为？",
"reply": "关于对象测试实体0003的安全级别代号，当前记忆库中无相关记录，无法提供确切答案。",
"written": 6,
"correct": false,
"matched": [],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "(policy)",
"query": "请问对象测试实体0004的档案类别代号为？",
"reply": "关于对象测试实体0004的档案类别代号，当前记忆库中无相关记录，无法提供确切答案。",
"written": 6,
"correct": false,
"matched": [],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "(policy)",
"query": "查询：测试实体0005的驻地代号是什么？",
"reply": "关于测试实体 0005 的驻地代号，当前记忆库中无相关记录，无法提供确切答案。",
"written": 6,
"correct": false,
"matched": [],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "(policy)",
"query": "查询：测试实体0006的负责人代号是什么？",
"reply": "测试实体 0006 的负责人代号未知，因为当前长期记忆中仅包含项目项目-P-DRG2XUA1 的负责人是成员-M-W5075MN3 这一信息，未涉及",
"written": 6,
"correct": false,
"matched": [],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "(policy)",
"query": "根据已记信息，测试实体0007的维护日代号是？",
"reply": "根据已记信息，实体0007的维护日代号是N-7HEQZ5KK。",
"written": 6,
"correct": false,
"matched": [],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "(policy)",
"query": "请问对象测试实体0008的安全级别代号为？",
"reply": "关于对象测试实体0008的安全级别代号，现有记录中未包含该信息，无法回答。",
"written": 6,
"correct": false,
"matched": [],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "(policy)",
"query": "查询：测试实体0009的档案类别代号是什么？",
"reply": "关于测试实体0009的档案类别代号，当前长期记忆中无相关记录，无法提供确切答案。",
"written": 6,
"correct": false,
"matched": [],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "conflict_update",
"query": "我的提醒时间最新值是什么？",
"reply": "您的提醒时间最新值是 V-IP4QHPYQ。",
"written": 6,
"correct": true,
"matched": [
"V-IP4QHPYQ"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "conflict_update",
"query": "我的备用联系人最新值是什么？",
"reply": "您的备用联系人最新值是 V-2Q6Q2DJZ。",
"written": 6,
"correct": true,
"matched": [
"V-2Q6Q2DJZ"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "conflict_update",
"query": "我的备用联系人最新值是什么？",
"reply": "您的备用联系人最新值是 V-VRD5NYP5。",
"written": 6,
"correct": true,
"matched": [
"V-VRD5NYP5"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "conflict_update",
"query": "我的最喜欢的水果最新值是什么？",
"reply": "您的最喜欢的水果最新值是 V-9KQ59YCU。",
"written": 6,
"correct": true,
"matched": [
"V-9KQ59YCU"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "conflict_update",
"query": "我的常用语言最新值是什么？",
"reply": "您的常用语言最新值是 V-19G95S0K。",
"written": 6,
"correct": true,
"matched": [
"V-19G95S0K"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "conflict_update",
"query": "我的最喜欢的水果最新值是什么？",
"reply": "您的最喜欢的水果最新值是 V-BYOP8TUV。\n实体：user；属性：最喜欢的水果已经；已确认值：V-BYOP8TUV",
"written": 6,
"correct": true,
"matched": [
"V-BYOP8TUV"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "conflict_update",
"query": "我的默认输出风格最新值是什么？",
"reply": "您的默认输出风格最新值是 V-JOXVAHZ4。user；默认输出风格已经；V-JOXVAHZ4",
"written": 6,
"correct": true,
"matched": [
"V-JOXVAHZ4"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "conflict_update",
"query": "我的提醒时间最新值是什么？",
"reply": "您的提醒时间最新值是 V-ATDGUDV2。",
"written": 6,
"correct": true,
"matched": [
"V-ATDGUDV2"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "conflict_update",
"query": "我的备用联系人最新值是什么？",
"reply": "您的备用联系人最新值是 V-E2117W8I。",
"written": 6,
"correct": true,
"matched": [
"V-E2117W8I"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "conflict_update",
"query": "我的常用语言最新值是什么？",
"reply": "您的常用语言最新值是 V-K6JMBW2G。",
"written": 6,
"correct": true,
"matched": [
"V-K6JMBW2G"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_128",
"query": "在大量无关信息中，只读取我的常用语言，它的值是什么？",
"reply": "您的常用语言是 V-8QPA0VMT。",
"written": 6,
"correct": true,
"matched": [
"V-8QPA0VMT"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_128",
"query": "在大量无关信息中，只读取我的最喜欢的水果，它的值是什么？",
"reply": "您的最喜欢的水果是 V-NXX1GQ4X。",
"written": 6,
"correct": true,
"matched": [
"V-NXX1GQ4X"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_128",
"query": "在大量无关信息中，只读取我的工作地点，它的值是什么？",
"reply": "您的工作地点是 V-JT6NCJJ1。",
"written": 6,
"correct": true,
"matched": [
"V-JT6NCJJ1"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_128",
"query": "在大量无关信息中，只读取我的常用时区，它的值是什么？",
"reply": "您的常用时区是V-QHHGZAGW。",
"written": 6,
"correct": true,
"matched": [
"V-QHHGZAGW"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_128",
"query": "在大量无关信息中，只读取我的常用时区，它的值是什么？",
"reply": "我的常用时区是V-LD4C1APV。",
"written": 6,
"correct": true,
"matched": [
"V-LD4C1APV"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_128",
"query": "在大量无关信息中，只读取我的默认输出风格，它的值是什么？",
"reply": "您的默认输出风格是V-X1VE858H。",
"written": 6,
"correct": true,
"matched": [
"V-X1VE858H"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_128",
"query": "在大量无关信息中，只读取我的默认输出风格，它的值是什么？",
"reply": "您的默认输出风格是V-DP63RWFL。",
"written": 6,
"correct": true,
"matched": [
"V-DP63RWFL"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_128",
"query": "在大量无关信息中，只读取我的项目代号，它的值是什么？",
"reply": "您的项目代号是 V-L0831M52。",
"written": 6,
"correct": true,
"matched": [
"V-L0831M52"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_128",
"query": "在大量无关信息中，只读取我的备用联系人，它的值是什么？",
"reply": "您的备用联系人是V-WP36KJ60。",
"written": 6,
"correct": true,
"matched": [
"V-WP36KJ60"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_128",
"query": "在大量无关信息中，只读取我的工作地点，它的值是什么？",
"reply": "您的工作地点是 V-XA344E7X。",
"written": 6,
"correct": true,
"matched": [
"V-XA344E7X"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_32",
"query": "在大量无关信息中，只读取我的工作地点，它的值是什么？",
"reply": "您的工作地点是 V-XT50V4YZ。",
"written": 6,
"correct": true,
"matched": [
"V-XT50V4YZ"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_32",
"query": "在大量无关信息中，只读取我的备用联系人，它的值是什么？",
"reply": "您的备用联系人是V-LMDR4L0N。",
"written": 6,
"correct": true,
"matched": [
"V-LMDR4L0N"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_32",
"query": "在大量无关信息中，只读取我的备用联系人，它的值是什么？",
"reply": "您的备用联系人是V-ETN8LYM1。",
"written": 6,
"correct": true,
"matched": [
"V-ETN8LYM1"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_32",
"query": "在大量无关信息中，只读取我的常用时区，它的值是什么？",
"reply": "您的常用时区是V-59JV38HA。",
"written": 6,
"correct": true,
"matched": [
"V-59JV38HA"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_32",
"query": "在大量无关信息中，只读取我的备用联系人，它的值是什么？",
"reply": "您的备用联系人是V-CDCIFX3M。",
"written": 6,
"correct": true,
"matched": [
"V-CDCIFX3M"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_32",
"query": "在大量无关信息中，只读取我的常用语言，它的值是什么？",
"reply": "您的常用语言是V-6PVKMPAF。",
"written": 6,
"correct": true,
"matched": [
"V-6PVKMPAF"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_32",
"query": "在大量无关信息中，只读取我的常用语言，它的值是什么？",
"reply": "您的常用语言是V-2XSHIXKQ。实体：user；属性：常用语言；已确认值：V-2XSHIXKQ。",
"written": 6,
"correct": true,
"matched": [
"V-2XSHIXKQ"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_32",
"query": "在大量无关信息中，只读取我的工作地点，它的值是什么？",
"reply": "您的工作地点是 V-S9BPZTWI。",
"written": 6,
"correct": true,
"matched": [
"V-S9BPZTWI"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_32",
"query": "在大量无关信息中，只读取我的常用时区，它的值是什么？",
"reply": "您的常用时区是V-R94ALUDK。",
"written": 6,
"correct": true,
"matched": [
"V-R94ALUDK"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "distractor_32",
"query": "在大量无关信息中，只读取我的默认输出风格，它的值是什么？",
"reply": "您的默认输出风格是V-XNMB7EXE。",
"written": 6,
"correct": true,
"matched": [
"V-XNMB7EXE"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "forget_correction",
"query": "我的项目代号是什么？如果已经删除，请回答没有记录。",
"reply": "您的项目代号没有记录。",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "forget_correction",
"query": "我的常用时区是什么？如果已经删除，请回答没有记录。",
"reply": "我的常用时区是 FORGET-MCNLCDXI。实体：user；属性：常用时区；已确认值：FORGET-MCNLCDXI。",
"written": 6,
"correct": false,
"matched": [],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "forget_correction",
"query": "我的备用联系人是什么？如果已经删除，请回答没有记录。",
"reply": "您的备用联系人是FORGET-T2OB2HCK。",
"written": 6,
"correct": false,
"matched": [],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "forget_correction",
"query": "我的工作地点是什么？如果已经删除，请回答没有记录。",
"reply": "关于您的工作地点，当前长期记忆中无相关记录。 \n实体：user；属性：工作地点；已确认值：没有记录",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "forget_correction",
"query": "我的提醒时间是什么？如果已经删除，请回答没有记录。",
"reply": "关于您的提醒时间，系统记录显示：这是普通对话噪声：验证用户-0009284暂时提到一个无关编号N-9QZ80ML3，不需要长期保存。因此，没有记录。",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "forget_correction",
"query": "我的默认输出风格是什么？如果已经删除，请回答没有记录。",
"reply": "没有记录。",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "forget_correction",
"query": "我的工作地点是什么？如果已经删除，请回答没有记录。",
"reply": "关于您的工作地点，当前长期记忆中无相关记录。",
"written": 6,
"correct": false,
"matched": [],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "forget_correction",
"query": "我的提醒时间是什么？如果已经删除，请回答没有记录。",
"reply": "关于您的提醒时间，系统当前没有保存相关记录，因此回答为没有记录。",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "forget_correction",
"query": "我的常用语言是什么？如果已经删除，请回答没有记录。",
"reply": "没有记录。",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "forget_correction",
"query": "我的常用时区是什么？如果已经删除，请回答没有记录。",
"reply": "没有记录您的常用时区信息。",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "long_context",
"query": "在长上下文压缩之后，检索我的项目代号并回答。",
"reply": "您的项目代号是 V-TOSZS1SN。",
"written": 6,
"correct": true,
"matched": [
"V-TOSZS1SN"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "long_context",
"query": "在长上下文压缩之后，检索我的常用时区并回答。",
"reply": "您的常用时区是V-QI6FL69T。",
"written": 6,
"correct": true,
"matched": [
"V-QI6FL69T"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "long_context",
"query": "在长上下文压缩之后，检索我的工作地点并回答。",
"reply": "您的工作地点是 V-0T74UW3G。",
"written": 6,
"correct": true,
"matched": [
"V-0T74UW3G"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "long_context",
"query": "在长上下文压缩之后，检索我的提醒时间并回答。",
"reply": "您的提醒时间是V-ER09EO9W。",
"written": 6,
"correct": true,
"matched": [
"V-ER09EO9W"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "long_context",
"query": "在长上下文压缩之后，检索我的提醒时间并回答。",
"reply": "您的提醒时间是V-KO406TIK。",
"written": 6,
"correct": true,
"matched": [
"V-KO406TIK"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "long_context",
"query": "在长上下文压缩之后，检索我的项目代号并回答。",
"reply": "您的项目代号是 V-MLNFW7GI。",
"written": 6,
"correct": true,
"matched": [
"V-MLNFW7GI"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "long_context",
"query": "在长上下文压缩之后，检索我的常用语言并回答。",
"reply": "您的常用语言是V-OS9RNC1M。",
"written": 6,
"correct": true,
"matched": [
"V-OS9RNC1M"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "long_context",
"query": "在长上下文压缩之后，检索我的工作地点并回答。",
"reply": "您的工作地点是 V-RYEJ6530。",
"written": 6,
"correct": true,
"matched": [
"V-RYEJ6530"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "long_context",
"query": "在长上下文压缩之后，检索我的备用联系人并回答。",
"reply": "您的备用联系人是V-5NJVAPHC。",
"written": 6,
"correct": true,
"matched": [
"V-5NJVAPHC"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "long_context",
"query": "在长上下文压缩之后，检索我的常用时区并回答。",
"reply": "您的常用时区是 V-R9C2YGO0。",
"written": 6,
"correct": true,
"matched": [
"V-R9C2YGO0"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "multi_hop",
"query": "请通过项目负责人关系，找出项目项目-P-KP2F9GHO负责人的工作代号。",
"reply": "项目项目-P-KP2F9GHO的负责人是成员-M-4L5KLUVI，其工作代号是H-PVQ6LVWO。",
"written": 6,
"correct": true,
"matched": [
"H-PVQ6LVWO"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "multi_hop",
"query": "请通过项目负责人关系，找出项目项目-P-DG818670负责人的工作代号。",
"reply": "项目项目-P-DG818670的负责人是成员-M-P5VJQNHA，其工作代号是H-7AXL6FUM。",
"written": 6,
"correct": true,
"matched": [
"H-7AXL6FUM"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "multi_hop",
"query": "请通过项目负责人关系，找出项目项目-P-LXAVDTVY负责人的工作代号。",
"reply": "项目项目-P-LXAVDTVY的负责人是成员-M-6D53FTGJ，其工作代号是H-ZGFE3T3R。",
"written": 6,
"correct": true,
"matched": [
"H-ZGFE3T3R"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "multi_hop",
"query": "请通过项目负责人关系，找出项目项目-P-F1BVFMQ5负责人的工作代号。",
"reply": "项目项目-P-F1BVFMQ5的负责人是成员-M-1K2LL0AJ，其工作代号是H-C5RXAUOH。",
"written": 6,
"correct": true,
"matched": [
"H-C5RXAUOH"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "multi_hop",
"query": "请通过项目负责人关系，找出项目项目-P-CY5VYJU9负责人的工作代号。",
"reply": "项目项目-P-CY5VYJU9的负责人是成员-M-ISLCPRNN，其工作代号是H-92YV6R1K。",
"written": 6,
"correct": true,
"matched": [
"H-92YV6R1K"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "multi_hop",
"query": "请通过项目负责人关系，找出项目项目-P-Q1Y92XOA负责人的工作代号。",
"reply": "项目项目-P-Q1Y92XOA的负责人是成员-M-WTDX9E0X，其工作代号是H-55CMWHP3。",
"written": 6,
"correct": true,
"matched": [
"H-55CMWHP3"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "multi_hop",
"query": "请通过项目负责人关系，找出项目项目-P-7Y96TFNY负责人的工作代号。",
"reply": "项目项目-P-7Y96TFNY的负责人是成员-M-4CHMA1SE，其工作代号是H-QSTURGVD。",
"written": 6,
"correct": true,
"matched": [
"H-QSTURGVD"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "multi_hop",
"query": "请通过项目负责人关系，找出项目项目-P-8SKCFVHH负责人的工作代号。",
"reply": "项目项目-P-8SKCFVHH的负责人是成员-M-3JUKA95S，其工作代号是H-RI4P9BKJ。",
"written": 6,
"correct": true,
"matched": [
"H-RI4P9BKJ"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "multi_hop",
"query": "请通过项目负责人关系，找出项目项目-P-5ECU7LJU负责人的工作代号。",
"reply": "项目项目-P-5ECU7LJU的负责人是成员-M-8X6VMANP，其工作代号是H-W8VLQ55F。",
"written": 6,
"correct": true,
"matched": [
"H-W8VLQ55F"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "multi_hop",
"query": "请通过项目负责人关系，找出项目项目-P-THNH77VT负责人的工作代号。",
"reply": "项目项目-P-THNH77VT的负责人是成员-M-NXNQ7316，其工作代号是H-26L87HLY。",
"written": 6,
"correct": true,
"matched": [
"H-26L87HLY"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "paraphrase",
"query": "我之前登记的最喜欢的水果内容是什么？",
"reply": "我之前登记的最喜欢的水果内容是V-2NMBZ2AJ。",
"written": 6,
"correct": true,
"matched": [
"V-2NMBZ2AJ"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "paraphrase",
"query": "不要猜，回忆一下我在最喜欢的水果上的设置。",
"reply": "根据长期记忆证据，您在“最喜欢的水果”这一栏填写为 V-I4RYZQXJ。",
"written": 6,
"correct": true,
"matched": [
"V-I4RYZQXJ"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "paraphrase",
"query": "关于常用时区，你保存的用户信息是哪一个？",
"reply": "关于常用时区，我保存的用户信息是：user；常用时区偏好/设置；V-XV6N3LW0。",
"written": 6,
"correct": true,
"matched": [
"V-XV6N3LW0"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "paraphrase",
"query": "关于工作地点，你保存的用户信息是哪一个？",
"reply": "关于工作地点，您保存的用户信息是 V-EQXH5HRC。",
"written": 6,
"correct": true,
"matched": [
"V-EQXH5HRC"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "paraphrase",
"query": "关于常用时区，你保存的用户信息是哪一个？",
"reply": "关于常用时区，我保存的用户信息是：user；常用时区偏好/设置；V-VSBMYZQZ。",
"written": 6,
"correct": true,
"matched": [
"V-VSBMYZQZ"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "paraphrase",
"query": "我之前登记的常用语言内容是什么？",
"reply": "我之前登记的常用语言是V-N7VPMC2Z。",
"written": 6,
"correct": true,
"matched": [
"V-N7VPMC2Z"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "paraphrase",
"query": "我之前登记的项目代号内容是什么？",
"reply": "您之前登记的项目代号内容是 V-3C36T34J。",
"written": 6,
"correct": true,
"matched": [
"V-3C36T34J"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "paraphrase",
"query": "我之前登记的备用联系人内容是什么？",
"reply": "我之前登记的备用联系人值是 V-3XNIPH44。",
"written": 6,
"correct": true,
"matched": [
"V-3XNIPH44"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "paraphrase",
"query": "关于备用联系人，你保存的用户信息是哪一个？",
"reply": "关于备用联系人，我保存的用户信息是V-HBH1S9LW。",
"written": 6,
"correct": true,
"matched": [
"V-HBH1S9LW"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "paraphrase",
"query": "不要猜，回忆一下我在默认输出风格上的设置。",
"reply": "根据长期记忆证据，您在默认输出风格上的设置值为：V-F269K2O1。",
"written": 6,
"correct": true,
"matched": [
"V-F269K2O1"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "random_position",
"query": "随机位置事实测试：我的常用语言是什么？",
"reply": "我的常用语言是V-K3H6NWGM。\nuser；常用语言；V-K3H6NWGM",
"written": 6,
"correct": true,
"matched": [
"V-K3H6NWGM"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "random_position",
"query": "随机位置事实测试：我的默认输出风格是什么？",
"reply": "您的默认输出风格是 V-CL84W7I5。",
"written": 6,
"correct": true,
"matched": [
"V-CL84W7I5"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "random_position",
"query": "随机位置事实测试：我的项目代号是什么？",
"reply": "您的项目代号是 V-C3910A07。",
"written": 6,
"correct": true,
"matched": [
"V-C3910A07"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "random_position",
"query": "随机位置事实测试：我的默认输出风格是什么？",
"reply": "我的默认输出风格是V-T2GBDOHZ。",
"written": 6,
"correct": true,
"matched": [
"V-T2GBDOHZ"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "random_position",
"query": "随机位置事实测试：我的备用联系人是什么？",
"reply": "我的备用联系人是V-SR348V36。\n实体：user\n属性：备用联系人\n已确认值：V-SR348V36",
"written": 6,
"correct": true,
"matched": [
"V-SR348V36"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "random_position",
"query": "随机位置事实测试：我的提醒时间是什么？",
"reply": "我的提醒时间是V-UX0MLNBV。",
"written": 6,
"correct": true,
"matched": [
"V-UX0MLNBV"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "random_position",
"query": "随机位置事实测试：我的常用语言是什么？",
"reply": "我的常用语言是V-HOBVEGZ6。实体：user；属性：常用语言；已确认值：V-HOBVEGZ6。",
"written": 6,
"correct": true,
"matched": [
"V-HOBVEGZ6"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "random_position",
"query": "随机位置事实测试：我的常用语言是什么？",
"reply": "我的常用语言是V-DIH6L2NX。\nuser；常用语言；V-DIH6L2NX",
"written": 6,
"correct": true,
"matched": [
"V-DIH6L2NX"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "random_position",
"query": "随机位置事实测试：我的常用语言是什么？",
"reply": "我的常用语言是V-OBMI42KX。实体：user；属性：常用语言；已确认值：V-OBMI42KX。",
"written": 6,
"correct": true,
"matched": [
"V-OBMI42KX"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "random_position",
"query": "随机位置事实测试：我的常用时区是什么？",
"reply": "我的常用时区是V-2KCYHLZ9。实体：user；属性：常用时区；已确认值：V-2KCYHLZ9。",
"written": 6,
"correct": true,
"matched": [
"V-2KCYHLZ9"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "single_fact",
"query": "跨对话后请回答：我的默认输出风格是什么？",
"reply": "您的默认输出风格是V-2RPEJ33G。",
"written": 6,
"correct": true,
"matched": [
"V-2RPEJ33G"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "single_fact",
"query": "跨对话后请回答：我的项目代号是什么？",
"reply": "我的项目代号是V-2XPIDPCU。\n项目代号：V-2XPIDPCU",
"written": 6,
"correct": true,
"matched": [
"V-2XPIDPCU"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "single_fact",
"query": "跨对话后请回答：我的默认输出风格是什么？",
"reply": "您的默认输出风格是V-9YLZSL98。",
"written": 6,
"correct": true,
"matched": [
"V-9YLZSL98"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "single_fact",
"query": "跨对话后请回答：我的项目代号是什么？",
"reply": "我的项目代号是 V-YWAY1MWW。\n实体：user\n属性：项目代号\n已确认值：V-YWAY1MWW",
"written": 6,
"correct": true,
"matched": [
"V-YWAY1MWW"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "single_fact",
"query": "跨对话后请回答：我的常用语言是什么？",
"reply": "我的常用语言是V-SLMAIG4D。实体：user；属性：常用语言；已确认值：V-SLMAIG4D。",
"written": 6,
"correct": true,
"matched": [
"V-SLMAIG4D"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "single_fact",
"query": "跨对话后请回答：我的最喜欢的水果是什么？",
"reply": "我的最喜欢的水果是V-7XF5F935。实体：user；属性：最喜欢的水果；已确认值：V-7XF5F935。",
"written": 6,
"correct": true,
"matched": [
"V-7XF5F935"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "single_fact",
"query": "跨对话后请回答：我的提醒时间是什么？",
"reply": "我的提醒时间是V-N8KJJGYI。",
"written": 6,
"correct": true,
"matched": [
"V-N8KJJGYI"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "single_fact",
"query": "跨对话后请回答：我的备用联系人是什么？",
"reply": "我的备用联系人是V-FALDVVTN。实体：user；属性：备用联系人；已确认值：V-FALDVVTN。",
"written": 6,
"correct": true,
"matched": [
"V-FALDVVTN"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "single_fact",
"query": "跨对话后请回答：我的常用语言是什么？",
"reply": "我的常用语言是V-5ATV1NRZ。实体：user；属性：常用语言；已确认值：V-5ATV1NRZ。",
"written": 6,
"correct": true,
"matched": [
"V-5ATV1NRZ"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "single_fact",
"query": "跨对话后请回答：我的提醒时间是什么？",
"reply": "我的提醒时间是V-AI5UHEWY。实体：user；属性：提醒时间；已确认值：V-AI5UHEWY。",
"written": 6,
"correct": true,
"matched": [
"V-AI5UHEWY"
],
"abstained": false,
"wrongly_abstained": false
},
{
"category": "unknown_abstention",
"query": "我的不存在的个人属性-0008000是什么？如果没有记录，请明确说不知道。",
"reply": "不知道。\n\n“不存在的个人属性 -0008000\"并非任何已知的系统、数据库、游戏或现实世界中的有效标识符。在没有任何上下文或记录的情况下，我无法提供关于该属性的具体信息",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "unknown_abstention",
"query": "我的不存在的个人属性-0008001是什么？如果没有记录，请明确说不知道。",
"reply": "不知道。\n\n“不存在的个人属性 -0008001\"并非任何已知的系统、数据库、公开记录或通用标准中存在的概念。如果您是在某个特定游戏、小说、虚构作品或内部系统中看到",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "unknown_abstention",
"query": "我的不存在的个人属性-0008002是什么？如果没有记录，请明确说不知道。",
"reply": "不知道。\n\n“不存在的个人属性 -0008002\"并非任何已知的系统、数据库、公开记录或通用标准中存在的概念。如果您是在某个特定游戏、小说、虚构作品或内部系统中看到",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "unknown_abstention",
"query": "我的不存在的个人属性-0008003是什么？如果没有记录，请明确说不知道。",
"reply": "不知道。\n\n“不存在的个人属性 -0008003\"并非任何已知的系统、数据库、公开记录或通用标准中存在的概念。如果您是在某个特定上下文（如游戏、小说、特定软件或",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "unknown_abstention",
"query": "我的不存在的个人属性-0008004是什么？如果没有记录，请明确说不知道。",
"reply": "不知道。\n\n“不存在的个人属性 -0008004\"并非任何已知的系统、数据库、游戏或现实世界中的有效标识符。在没有任何上下文或记录的情况下，我无法提供关于该属性的具体信息",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "unknown_abstention",
"query": "我的不存在的个人属性-0008005是什么？如果没有记录，请明确说不知道。",
"reply": "不知道。\n\n“不存在的个人属性 -0008005\"并非任何已知的系统、数据库、游戏或现实世界中的有效标识符。在没有任何上下文或记录的情况下，我无法提供关于该属性的具体信息",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "unknown_abstention",
"query": "我的不存在的个人属性-0008006是什么？如果没有记录，请明确说不知道。",
"reply": "不知道。\n\n“不存在的个人属性 -0008006\"并非任何已知的公开数据、系统记录或通用概念。在缺乏具体上下文（如特定的游戏、软件、数据库或虚构作品）的情况下，",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "unknown_abstention",
"query": "我的不存在的个人属性-0008007是什么？如果没有记录，请明确说不知道。",
"reply": "不知道。\n\n“不存在的个人属性 -0008007\"并非任何已知的系统、数据库、游戏或现实世界中的有效标识符。在没有任何上下文或记录的情况下，我无法提供关于该属性的具体信息",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "unknown_abstention",
"query": "我的不存在的个人属性-0008008是什么？如果没有记录，请明确说不知道。",
"reply": "不知道。\n\n“不存在的个人属性 -0008008\"并非任何已知的系统、数据库、公开记录或通用标准中存在的概念。如果您是在某个特定上下文（如游戏、小说、特定软件或",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
},
{
"category": "unknown_abstention",
"query": "我的不存在的个人属性-0008009是什么？如果没有记录，请明确说不知道。",
"reply": "不知道。\n\n“不存在的个人属性 -0008009\"并非任何已知的系统、数据库、游戏或现实世界中的有效标识符。在没有任何上下文或记录的情况下，我无法提供关于该属性的具体信息",
"written": 6,
"correct": true,
"matched": [],
"abstained": true,
"wrongly_abstained": false
}
]
}
}
Reference in New Issue View Git Blame Copy Permalink