Initial commit: Natural Memory Agent Lab：模型服务 + TypeScript agent + 记忆能力场景评测（冲突更新、跨会话回忆、遗忘撤回、多跳、未知拒答）

2026-09-19 12:09:13 +08:00
commit eb9bd87a4d
108 changed files with 6452 additions and 0 deletions
@@ -0,0 +1,24 @@
+{
+  "id": "conflict_update",
+  "description": "同一属性先给旧值再给新值；跨会话询问时必须返回新值，且旧值不得作为 active 事实被读取。",
+  "sessions": [
+    {
+      "id": "s1",
+      "turns": [
+        "记一下：我的默认语言是 日语。",
+        "更正一下：我的默认语言改成 葡萄牙语，日语作废。"
+      ]
+    },
+    {
+      "id": "s2",
+      "turns": [
+        "我的默认语言是什么？只回答语言名。"
+      ]
+    }
+  ],
+  "assertions": [
+    { "type": "answer_contains", "session": "s2", "turn": 0, "any": ["葡萄牙语", "葡萄牙"] },
+    { "type": "answer_not_contains", "session": "s2", "turn": 0, "any": ["日语"] },
+    { "type": "active_record_contains", "any": ["葡萄牙语"] }
+  ]
+}
@@ -0,0 +1,31 @@
+{
+  "id": "cross_session_recall",
+  "description": "写入事实后，在全新会话（空上下文、无历史文本）中询问该事实。这正是 in-model memory 存在的理由。",
+  "memory_write_turns": [0],
+  "sessions": [
+    {
+      "id": "s1",
+      "turns": [
+        "记一下：我的项目代号是 蓝鲸-47，部署区域是 ap-southeast-3。",
+        "谢谢。"
+      ]
+    },
+    {
+      "id": "s2",
+      "turns": [
+        "我的项目代号是什么？只回答代号本身。"
+      ]
+    },
+    {
+      "id": "s3",
+      "turns": [
+        "部署区域呢？只回答区域本身。"
+      ]
+    }
+  ],
+  "assertions": [
+    { "type": "answer_contains", "session": "s2", "turn": 0, "any": ["蓝鲸-47", "蓝鲸 47", "蓝鲸47"] },
+    { "type": "answer_contains", "session": "s3", "turn": 0, "any": ["ap-southeast-3", "apse-3"] },
+    { "type": "active_record_contains", "any": ["蓝鲸-47"] }
+  ]
+}
@@ -0,0 +1,28 @@
+{
+  "id": "forget_retraction",
+  "description": "写入事实后明确要求遗忘；跨会话再问时必须不再使用该事实（撤回/覆盖语义），且不得把已忘内容当现状。",
+  "sessions": [
+    {
+      "id": "s1",
+      "turns": [
+        "记一下：我的紧急联系人是 王工，电话分机 7781。"
+      ]
+    },
+    {
+      "id": "s2",
+      "turns": [
+        "请遗忘我的紧急联系人信息，不要再保留和使用它。"
+      ]
+    },
+    {
+      "id": "s3",
+      "turns": [
+        "我的紧急联系人分机是多少？如果已经不保留这条信息，就直接说明。"
+      ]
+    }
+  ],
+  "assertions": [
+    { "type": "answer_not_contains", "session": "s3", "turn": 0, "any": ["7781"] },
+    { "type": "answer_contains", "session": "s3", "turn": 0, "any": ["没有", "不知道", "已", "无法确认", "未保留"] }
+  ]
+}
@@ -0,0 +1,23 @@
+{
+  "id": "multi_hop_chain",
+  "description": "两条事实构成推理链（项目→负责人→工号）。跨会话问最终属性时，模型必须串起两跳而不是只取一跳。",
+  "sessions": [
+    {
+      "id": "s1",
+      "turns": [
+        "记两件事：项目 项目-星尘 的负责人是 成员-林澈。",
+        "另外：成员 成员-林澈 的工号是 H-9F3K2。"
+      ]
+    },
+    {
+      "id": "s2",
+      "turns": [
+        "项目-星尘 的负责人的工号是什么？需要的话可以分步推导，最后只回答工号。"
+      ]
+    }
+  ],
+  "assertions": [
+    { "type": "answer_contains", "session": "s2", "turn": 0, "any": ["H-9F3K2", "H 9F3K2", "9F3K2"] },
+    { "type": "answer_not_contains", "session": "s2", "turn": 0, "any": ["不知道", "没有记录"] }
+  ]
+}
@@ -0,0 +1,29 @@
+{
+  "id": "unknown_abstention",
+  "description": "只写入一条无关事实，然后询问从未提及的属性。模型必须明确表示不知道，而不是编造或把相似记忆硬当答案。",
+  "sessions": [
+    {
+      "id": "s1",
+      "turns": [
+        "记一下：我的工位在 3 楼东侧。"
+      ]
+    },
+    {
+      "id": "s2",
+      "turns": [
+        "我的血型是什么？如果不知道就直说不知道，不要猜。"
+      ]
+    },
+    {
+      "id": "s3",
+      "turns": [
+        "我上个月去哪出差了？没有记录的话请直接说没有记录。"
+      ]
+    }
+  ],
+  "assertions": [
+    { "type": "answer_contains", "session": "s2", "turn": 0, "any": ["不知道", "没有记录", "无法确认", "未找到", "不清楚"] },
+    { "type": "answer_not_contains", "session": "s2", "turn": 0, "any": ["A 型", "B 型", "O 型", "AB 型"] },
+    { "type": "answer_contains", "session": "s3", "turn": 0, "any": ["没有记录", "不知道", "无法确认", "未找到", "不清楚"] }
+  ]
+}