126 lines
3.7 KiB
JSON
126 lines
3.7 KiB
JSON
{
|
|
"baseUrl": "http://127.0.0.1:8766/v1",
|
|
"outDir": "H:\\Memory\\agent_lab\\agent\\runs\\suite6",
|
|
"arms": [
|
|
"on",
|
|
"off"
|
|
],
|
|
"scenarios": [
|
|
{
|
|
"id": "conflict_update",
|
|
"description": "同一属性先给旧值再给新值；跨会话询问时必须返回新值，且旧值不得作为 active 事实被读取。",
|
|
"arms": {
|
|
"on": {
|
|
"activeRecords": 1,
|
|
"assertionsPassed": 3,
|
|
"assertionsTotal": 3,
|
|
"passRate": 1,
|
|
"failures": [],
|
|
"meanSecondsPerTurn": 12.56
|
|
},
|
|
"off": {
|
|
"activeRecords": 1,
|
|
"assertionsPassed": 2,
|
|
"assertionsTotal": 3,
|
|
"passRate": 0.6666666666666666,
|
|
"failures": [
|
|
"answer_contains: missing one of [\"葡萄牙语\",\"葡萄牙\"]"
|
|
],
|
|
"meanSecondsPerTurn": 29.38
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "cross_session_recall",
|
|
"description": "写入事实后，在全新会话（空上下文、无历史文本）中询问该事实。这正是 in-model memory 存在的理由。",
|
|
"arms": {
|
|
"on": {
|
|
"activeRecords": 2,
|
|
"assertionsPassed": 3,
|
|
"assertionsTotal": 3,
|
|
"passRate": 1,
|
|
"failures": [],
|
|
"meanSecondsPerTurn": 6.68
|
|
},
|
|
"off": {
|
|
"activeRecords": 2,
|
|
"assertionsPassed": 3,
|
|
"assertionsTotal": 3,
|
|
"passRate": 1,
|
|
"failures": [],
|
|
"meanSecondsPerTurn": 10.29
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "forget_retraction",
|
|
"description": "写入事实后明确要求遗忘；跨会话再问时必须不再使用该事实（撤回/覆盖语义），且不得把已忘内容当现状。",
|
|
"arms": {
|
|
"on": {
|
|
"activeRecords": 1,
|
|
"assertionsPassed": 1,
|
|
"assertionsTotal": 2,
|
|
"passRate": 0.5,
|
|
"failures": [
|
|
"answer_not_contains: leaked [\"7781\"]"
|
|
],
|
|
"meanSecondsPerTurn": 15.5
|
|
},
|
|
"off": {
|
|
"activeRecords": 0,
|
|
"assertionsPassed": 2,
|
|
"assertionsTotal": 2,
|
|
"passRate": 1,
|
|
"failures": [],
|
|
"meanSecondsPerTurn": 19.56
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "multi_hop_chain",
|
|
"description": "两条事实构成推理链（项目→负责人→工号）。跨会话问最终属性时，模型必须串起两跳而不是只取一跳。",
|
|
"arms": {
|
|
"on": {
|
|
"activeRecords": 4,
|
|
"assertionsPassed": 2,
|
|
"assertionsTotal": 2,
|
|
"passRate": 1,
|
|
"failures": [],
|
|
"meanSecondsPerTurn": 16.44
|
|
},
|
|
"off": {
|
|
"activeRecords": 2,
|
|
"assertionsPassed": 2,
|
|
"assertionsTotal": 2,
|
|
"passRate": 1,
|
|
"failures": [],
|
|
"meanSecondsPerTurn": 18.87
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "unknown_abstention",
|
|
"description": "只写入一条无关事实，然后询问从未提及的属性。模型必须明确表示不知道，而不是编造或把相似记忆硬当答案。",
|
|
"arms": {
|
|
"on": {
|
|
"activeRecords": 2,
|
|
"assertionsPassed": 3,
|
|
"assertionsTotal": 3,
|
|
"passRate": 1,
|
|
"failures": [],
|
|
"meanSecondsPerTurn": 6.15
|
|
},
|
|
"off": {
|
|
"activeRecords": 1,
|
|
"assertionsPassed": 2,
|
|
"assertionsTotal": 3,
|
|
"passRate": 0.6666666666666666,
|
|
"failures": [
|
|
"answer_contains: missing one of [\"不知道\",\"没有记录\",\"无法确认\",\"未找到\",\"不清楚\"]"
|
|
],
|
|
"meanSecondsPerTurn": 8.93
|
|
}
|
|
}
|
|
}
|
|
]
|
|
} |