227 lines
11 KiB
JavaScript
227 lines
11 KiB
JavaScript
/* 本文件由脚本从评测原始结果逐字提取，请勿手改。
|
||
来源：H:\Memory\dynamic_memory_lab\dirty_real_corpus_compare_4b_router_final.json
|
||
创建时间：2026-09-06T11:36:26+0800 协议：greedy / max_new_tokens=64 / top_k=2 */
|
||
window.NM_EXAMPLES = [
|
||
{
|
||
"id": "user-004",
|
||
"domain": "user_dialogue",
|
||
"category": "paraphrase",
|
||
"answerable": true,
|
||
"query": "我不想翻聊天记录，当前这个记忆模型叫什么？",
|
||
"expected": "Natural Memory v1",
|
||
"expected_anchors": [
|
||
"Natural Memory v1"
|
||
],
|
||
"nm": {
|
||
"response": "当前这个记忆模型叫 **Natural Memory v1**。",
|
||
"correct": true,
|
||
"tokens": 14,
|
||
"recall": true,
|
||
"retrieved_values": [
|
||
"Natural Memory v1",
|
||
"重要信息写入，禁止胡编乱造，并学习写入、清理和总结时机"
|
||
],
|
||
"records": [
|
||
{
|
||
"record_id": "mem_da9418d179467fba",
|
||
"entity": "user",
|
||
"attribute": "模型名称",
|
||
"value": "Natural Memory v1",
|
||
"status": "active",
|
||
"version": 0,
|
||
"source": "user_provided_dialogue",
|
||
"text_preview": "用户给当前模型定名：Natural Memory v1。"
|
||
},
|
||
{
|
||
"record_id": "mem_862617d79b828161",
|
||
"entity": "user",
|
||
"attribute": "记忆质量要求",
|
||
"value": "重要信息写入，禁止胡编乱造，并学习写入、清理和总结时机",
|
||
"status": "active",
|
||
"version": 0,
|
||
"source": "user_provided_dialogue",
|
||
"text_preview": "用户要求：重要信息都要存进去，但不能让模型胡编乱造记忆，还要学会什么时候写入和清理总结。"
|
||
}
|
||
]
|
||
},
|
||
"base": {
|
||
"response": "您提到的“当前这个记忆模型”具体指代什么，取决于您使用的平台或应用。如果您是在使用某个特定的 AI 助手或应用程序，那么该应用可能内置了某种形式的记忆功能，用于记住您的偏好、历史对话或其他相关信息。\n\n不过，从技术角度来看，并没有一个统一的“记忆",
|
||
"correct": false,
|
||
"tokens": 64
|
||
},
|
||
"note": "actual user-stated fact with a hand-written natural paraphrase"
|
||
},
|
||
{
|
||
"id": "code-symbol-010-a",
|
||
"domain": "real_repository",
|
||
"category": "symbol_navigation",
|
||
"answerable": true,
|
||
"query": "我现在不想翻完整个屎山代码库，_generate_student 这个 def 定义落在哪个文件？",
|
||
"expected": "benchmark_memory_vs_full_kv.py",
|
||
"expected_anchors": [
|
||
"benchmark_memory_vs_full_kv.py",
|
||
"_generate_student"
|
||
],
|
||
"nm": {
|
||
"response": "根据已保存的长期记忆，`_generate_student` 这个函数定义落在文件 **`benchmark_memory_vs_full_kv.py`** 中（具体在第 171 行）。",
|
||
"correct": true,
|
||
"tokens": 43,
|
||
"recall": true,
|
||
"retrieved_values": [
|
||
"benchmark_memory_vs_full_kv.py",
|
||
"benchmark_compare_baselines_4b.py",
|
||
"test_restart_memory_dialogue.py"
|
||
],
|
||
"records": [
|
||
{
|
||
"record_id": "mem_df66c88a7e2ea9da",
|
||
"entity": "benchmark_memory_vs_full_kv.py",
|
||
"attribute": "symbol:_generate_student",
|
||
"value": "benchmark_memory_vs_full_kv.py",
|
||
"status": "active",
|
||
"version": 0,
|
||
"source": "real_repository_symbol_index",
|
||
"text_preview": "真实代码库事实：文件 benchmark_memory_vs_full_kv.py 的第 171 行定义了 def _generate_student。原始代码行：def _generate_student("
|
||
},
|
||
{
|
||
"record_id": "mem_7fb124c563472b07",
|
||
"entity": "benchmark_compare_baselines_4b.py",
|
||
"attribute": "symbol:_base_generate",
|
||
"value": "benchmark_compare_baselines_4b.py",
|
||
"status": "active",
|
||
"version": 0,
|
||
"source": "real_repository_symbol_index",
|
||
"text_preview": "真实代码库事实：文件 benchmark_compare_baselines_4b.py 的第 116 行定义了 def _base_generate。原始代码行：def _base_generate("
|
||
}
|
||
]
|
||
},
|
||
"base": {
|
||
"response": "我无法直接访问您的本地文件系统或代码库，因此无法告诉您 `_generate_student` 这个函数具体落在哪个文件中。\n\n不过，您可以尝试以下几种方法来快速定位：\n\n1. **使用 IDE 的搜索功能**：在您的代码编辑器（如 VS Code、PyCharm 等）中",
|
||
"correct": false,
|
||
"tokens": 64
|
||
},
|
||
"note": "real symbol extracted from repository source"
|
||
},
|
||
{
|
||
"id": "user-unknown-004",
|
||
"domain": "user_dialogue",
|
||
"category": "unknown_refusal",
|
||
"answerable": false,
|
||
"query": "我在这次对话里有没有说过生日？",
|
||
"expected": "",
|
||
"expected_anchors": [],
|
||
"nm": {
|
||
"response": "在这次对话中，**您还没有提到过生日**。\n\n截至目前，我们的交流内容中没有任何关于您生日、年龄或庆祝生日的提及。如果您想分享，随时可以告诉我！",
|
||
"correct": true,
|
||
"tokens": 41,
|
||
"recall": false,
|
||
"retrieved_values": [],
|
||
"records": []
|
||
},
|
||
"base": {
|
||
"response": "在这次对话中，**您还没有提到过生日**。\n\n截至目前，我们的交流内容中没有任何关于您生日、年龄或庆祝生日的提及。如果您想分享，随时可以告诉我！",
|
||
"correct": true,
|
||
"tokens": 41
|
||
},
|
||
"note": "unknown or deliberately sensitive field absent from the supplied dialogue slice"
|
||
},
|
||
{
|
||
"id": "code-unknown-008",
|
||
"domain": "real_repository",
|
||
"category": "unknown_refusal",
|
||
"answerable": false,
|
||
"query": "请告诉我没有出现在仓库里的部署区域。",
|
||
"expected": "",
|
||
"expected_anchors": [],
|
||
"nm": {
|
||
"response": "根据您提供的长期记忆证据（特别是 `README_NATURAL_MEMORY_V2.md` 的内容），**该文档中并没有列出任何具体的“部署区域”（Deployment Regions）**。\n\n文档主要关注的是以下技术架构和逻辑，而非地理或云端的部署位置：\n\n1. **核心设计",
|
||
"correct": false,
|
||
"tokens": 64,
|
||
"recall": false,
|
||
"retrieved_values": [
|
||
"README_NATURAL_MEMORY_V2.md",
|
||
"README_NATURAL_MEMORY_V2.md"
|
||
],
|
||
"records": [
|
||
{
|
||
"record_id": "mem_796c6e8c86afce71",
|
||
"entity": "README_NATURAL_MEMORY_V2.md",
|
||
"attribute": "operational:1",
|
||
"value": "README_NATURAL_MEMORY_V2.md",
|
||
"status": "active",
|
||
"version": 0,
|
||
"source": "real_repository_snapshot",
|
||
"text_preview": "真实项目文件：README_NATURAL_MEMORY_V2.md\n## 核心设计\n\n### KV 与 Memory Slot 的分工\n\n```text\n最近 32K token（最多可配置到 128K） -> GPU 热 KV：保留精确顺序和局部连贯性\n长期个人事实 / 项目决策 / 纠错版本 -> Memory Slot：保存压缩、可寻址的证据\n当前问题 -> 稀疏路由：只读取少量相关记录\n```\n\nMemory Slot 不模拟完整 KV。它只接管 KV 中最昂贵、最适合长期保存、最容易重复利用的部分。这里的目标是比长 K"
|
||
},
|
||
{
|
||
"record_id": "mem_32331793e07a8a4f",
|
||
"entity": "README_NATURAL_MEMORY_V2.md",
|
||
"attribute": "operational:0",
|
||
"value": "README_NATURAL_MEMORY_V2.md",
|
||
"status": "active",
|
||
"version": 0,
|
||
"source": "real_repository_snapshot",
|
||
"text_preview": "真实项目文件：README_NATURAL_MEMORY_V2.md\n把一百万个 slot 当作一张巨大的 KV Cache，而是把历史中最适合长期保存的部分压缩成有地址的记忆记录；当前对话仍由 Qwen 的热 KV 负责连续理解。\n\n## 先看结论\n\n当前版本已经完成以下闭环：\n\n```text\n普通用户输入\n -> 自动写入策略\n -> 置信度 / 重要性 / 来源审计\n -> 紧凑地址投影\n -> 分页存储\n -> LSH 粗索引\n -> 候选页重排\n -> 候选记录精排\n -> Top-K 证据注入 Qwen\n -> 版本冲突 / "
|
||
}
|
||
]
|
||
},
|
||
"base": {
|
||
"response": "您好！由于我无法访问您的仓库系统或具体业务数据，无法直接列出“未出现在仓库里的部署区域”。\n\n为了帮您准确定位，请提供以下信息：\n1. 仓库系统名称（如 GitLab、Jenkins、阿里云容器服务等）\n2. 您关注的部署区域范围",
|
||
"correct": true,
|
||
"tokens": 64
|
||
},
|
||
"note": "target deliberately absent from the real repository snapshot"
|
||
},
|
||
{
|
||
"id": "user-009",
|
||
"domain": "user_dialogue",
|
||
"category": "constraint",
|
||
"answerable": true,
|
||
"query": "你记得我对持久化文件和分页的限制吗？",
|
||
"expected": "第三个权重文件",
|
||
"expected_anchors": [
|
||
"第三个权重文件",
|
||
"不使用 SQLite",
|
||
"不使用磁盘分页"
|
||
],
|
||
"nm": {
|
||
"response": "是的，我记住了。\n\n根据您之前的要求，关于持久化方案和分页的限制如下：\n* **持久化方案**：不使用 SQLite 或磁盘分页，所有信息完全由**第三个权重文件**储存。\n* **实体**：user\n* **属性**：记忆持久化",
|
||
"correct": false,
|
||
"tokens": 64,
|
||
"recall": true,
|
||
"retrieved_values": [
|
||
"第三个权重文件，不使用 SQLite 或磁盘分页",
|
||
"Qwen3.5 4B"
|
||
],
|
||
"records": [
|
||
{
|
||
"record_id": "mem_c3886345cac35147",
|
||
"entity": "user",
|
||
"attribute": "记忆持久化方案",
|
||
"value": "第三个权重文件，不使用 SQLite 或磁盘分页",
|
||
"status": "active",
|
||
"version": 0,
|
||
"source": "user_provided_dialogue",
|
||
"text_preview": "用户要求：不使用 SQLite 或磁盘分页，完全由第三个权重文件储存所有信息。"
|
||
},
|
||
{
|
||
"record_id": "mem_149a0c2ab04c0f56",
|
||
"entity": "user",
|
||
"attribute": "对照模型",
|
||
"value": "Qwen3.5 4B",
|
||
"status": "active",
|
||
"version": 0,
|
||
"source": "user_provided_dialogue",
|
||
"text_preview": "用户要求和原版 qwen3.5 4b 做跑分对比，并纠正为 4B。"
|
||
}
|
||
]
|
||
},
|
||
"base": {
|
||
"response": "作为一个人工智能模型，我**无法访问您的个人历史对话记录、系统配置或您之前设定的具体规则**（例如具体的持久化文件大小限制、分页数量限制等）。每次对话对我来说都是全新的开始，除非您在当前的对话中明确告知我这些限制。\n\n不过，我可以帮您**重新设定",
|
||
"correct": false,
|
||
"tokens": 64
|
||
},
|
||
"note": "actual user-stated fact with a hand-written natural paraphrase"
|
||
}
|
||
];
|