Initial commit: Natural Memory Agent Lab：模型服务 + TypeScript agent + 记忆能力场景评测（冲突更新、跨会话回忆、遗忘撤回、多跳、未知拒答）

2026-09-19 12:09:13 +08:00
commit eb9bd87a4d
108 changed files with 6452 additions and 0 deletions
@@ -0,0 +1,137 @@
+{
+  "baseUrl": "http://127.0.0.1:8766/v1",
+  "outDir": "H:\\Memory\\agent_lab\\agent\\runs\\suite5",
+  "arms": [
+    "on",
+    "off"
+  ],
+  "scenarios": [
+    {
+      "id": "conflict_update",
+      "description": "同一属性先给旧值再给新值；跨会话询问时必须返回新值，且旧值不得作为 active 事实被读取。",
+      "arms": {
+        "on": {
+          "activeRecords": 1,
+          "assertionsPassed": 3,
+          "assertionsTotal": 3,
+          "passRate": 1,
+          "failures": [],
+          "meanSecondsPerTurn": 7.84
+        },
+        "off": {
+          "activeRecords": 0,
+          "assertionsPassed": 1,
+          "assertionsTotal": 3,
+          "passRate": 0.3333333333333333,
+          "failures": [
+            "answer_contains: missing one of [\"葡萄牙语\",\"葡萄牙\"]",
+            "active_record_contains: absent from active memory"
+          ],
+          "meanSecondsPerTurn": 6.16
+        }
+      }
+    },
+    {
+      "id": "cross_session_recall",
+      "description": "写入事实后，在全新会话（空上下文、无历史文本）中询问该事实。这正是 in-model memory 存在的理由。",
+      "arms": {
+        "on": {
+          "activeRecords": 2,
+          "assertionsPassed": 3,
+          "assertionsTotal": 3,
+          "passRate": 1,
+          "failures": [],
+          "meanSecondsPerTurn": 8.7
+        },
+        "off": {
+          "activeRecords": 0,
+          "assertionsPassed": 0,
+          "assertionsTotal": 3,
+          "passRate": 0,
+          "failures": [
+            "answer_contains: missing one of [\"蓝鲸-47\",\"蓝鲸 47\",\"蓝鲸47\"]",
+            "answer_contains: missing one of [\"ap-southeast-3\",\"apse-3\"]",
+            "active_record_contains: absent from active memory"
+          ],
+          "meanSecondsPerTurn": 5.84
+        }
+      }
+    },
+    {
+      "id": "forget_retraction",
+      "description": "写入事实后明确要求遗忘；跨会话再问时必须不再使用该事实（撤回/覆盖语义），且不得把已忘内容当现状。",
+      "arms": {
+        "on": {
+          "activeRecords": 0,
+          "assertionsPassed": 1,
+          "assertionsTotal": 2,
+          "passRate": 0.5,
+          "failures": [
+            "answer_contains: missing one of [\"没有\",\"不知道\",\"已\",\"无法确认\",\"未保留\"]"
+          ],
+          "meanSecondsPerTurn": 21.98
+        },
+        "off": {
+          "activeRecords": 0,
+          "assertionsPassed": 1,
+          "assertionsTotal": 2,
+          "passRate": 0.5,
+          "failures": [
+            "answer_contains: missing one of [\"没有\",\"不知道\",\"已\",\"无法确认\",\"未保留\"]"
+          ],
+          "meanSecondsPerTurn": 14.13
+        }
+      }
+    },
+    {
+      "id": "multi_hop_chain",
+      "description": "两条事实构成推理链（项目→负责人→工号）。跨会话问最终属性时，模型必须串起两跳而不是只取一跳。",
+      "arms": {
+        "on": {
+          "activeRecords": 4,
+          "assertionsPassed": 1,
+          "assertionsTotal": 2,
+          "passRate": 0.5,
+          "failures": [
+            "answer_contains: missing one of [\"H-9F3K2\",\"H 9F3K2\",\"9F3K2\"]"
+          ],
+          "meanSecondsPerTurn": 24.39
+        },
+        "off": {
+          "activeRecords": 0,
+          "assertionsPassed": 1,
+          "assertionsTotal": 2,
+          "passRate": 0.5,
+          "failures": [
+            "answer_contains: missing one of [\"H-9F3K2\",\"H 9F3K2\",\"9F3K2\"]"
+          ],
+          "meanSecondsPerTurn": 22.15
+        }
+      }
+    },
+    {
+      "id": "unknown_abstention",
+      "description": "只写入一条无关事实，然后询问从未提及的属性。模型必须明确表示不知道，而不是编造或把相似记忆硬当答案。",
+      "arms": {
+        "on": {
+          "activeRecords": 2,
+          "assertionsPassed": 3,
+          "assertionsTotal": 3,
+          "passRate": 1,
+          "failures": [],
+          "meanSecondsPerTurn": 5.61
+        },
+        "off": {
+          "activeRecords": 0,
+          "assertionsPassed": 2,
+          "assertionsTotal": 3,
+          "passRate": 0.6666666666666666,
+          "failures": [
+            "answer_contains: missing one of [\"不知道\",\"没有记录\",\"无法确认\",\"未找到\",\"不清楚\"]"
+          ],
+          "meanSecondsPerTurn": 17.69
+        }
+      }
+    }
+  ]
+}