325 lines
15 KiB
Python
325 lines
15 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
Natural Memory v2 — 站点构建脚本
|
||
|
||
两种版式：
|
||
release —— 发布页（总览）：无侧栏、全宽分段，按研究/模型发布页的通行骨架组织
|
||
docs —— 文档页（其余 9 页）：左侧栏 + 窄栏正文
|
||
|
||
把 `src/pages/*.body.html` 的正文片段套进 `templates/base.html` 骨架；
|
||
正文里可用 `{{include:partials/xxx.html}}` 引入 `src/partials/` 下的共用片段。
|
||
|
||
用法：
|
||
python build.py # 生成全部页面
|
||
python build.py --check # 只校验，不写文件
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
from datetime import datetime
|
||
|
||
ROOT = os.path.dirname(os.path.abspath(__file__))
|
||
SRC = os.path.join(ROOT, "src", "pages")
|
||
PARTIALS = os.path.join(ROOT, "src", "partials")
|
||
TPL = os.path.join(ROOT, "templates", "base.html")
|
||
|
||
# slug, 序号, 中文标题, 英文小标, 分组, <title>, 摘要
|
||
PAGES = [
|
||
("index", "00", "总览", "OVERVIEW", "概览",
|
||
"Natural Memory v2 — 模型内的记忆层",
|
||
"接在本地 Qwen3.5-4B 上的模型内记忆层。模型重启后不携带历史聊天记录，仍能访问已保存的个人与项目事实，且不把全部历史转换成 GPU 上的长 KV Cache。"),
|
||
("division", "01", "记忆与 KV 的分工", "DIVISION OF LABOUR", "原理",
|
||
"记忆与 KV 的分工 — Natural Memory v2",
|
||
"为什么不是简单加长上下文：哪些东西该留在 Qwen 的热 KV 里，哪些该写入带地址的记忆记录，各自的边界在哪里。"),
|
||
("mechanism", "02", "一次读取与一次写入", "MECHANISM", "原理",
|
||
"机制：读取路径、记录结构与写入状态机 — Natural Memory v2",
|
||
"从用户问题到证据前缀注入的八个环节，记录包含哪些字段，以及 active / superseded / quarantined / retracted 四个状态如何演化。"),
|
||
("results", "03", "测量结果", "MEASUREMENTS", "证据",
|
||
"测量结果 — Natural Memory v2",
|
||
"同一 264 题题集的两个独立批次、结构化工程基准与记忆路由器工程线的全部实测数字，均标注出处与口径。"),
|
||
("nm21", "04", "NM2.1 · 之后做了什么", "WHAT CHANGED NEXT", "证据",
|
||
"NM2.1：之后做了什么 — Natural Memory v2",
|
||
"三处改动各自的测量依据、把端到端正确率锁死在 50% 的写入缺陷如何定位、75% 未知泄漏如何修到 0%，以及它接上运行时连续失败两次的完整记录。"),
|
||
("examples", "05", "实测样例", "GROUND TRUTH", "证据",
|
||
"实测样例 — Natural Memory v2",
|
||
"五条逐字取自评测原始结果的问答对照，包含两条我们判错的样例。"),
|
||
("ledger", "06", "真实评测台账", "FULL LEDGER", "证据",
|
||
"真实评测台账 — Natural Memory v2",
|
||
"全部 108 条真实评测 episode 的逐条台账：模型逐字回复、判定结果、停止原因、选中记录数与分数，可按来源与判定筛选。"),
|
||
("cost", "07", "代价账本", "COST", "工程",
|
||
"代价账本 — Natural Memory v2",
|
||
"记忆带来的延迟、解码速度、前缀长度与显存开销，以及显存快照的口径说明。"),
|
||
("capacity", "08", "容量与存储", "CAPACITY & STORAGE", "工程",
|
||
"容量与存储 — Natural Memory v2",
|
||
"地址空间、分页结构、有界 GPU cache 与嵌入式 weight-shard 持久化的实际数字与取舍。"),
|
||
("limits", "09", "我们不宣称的内容", "BOUNDARIES", "工程",
|
||
"边界：我们不宣称的内容 — Natural Memory v2",
|
||
"没有足够证据支持的八条说法、五项主要局限，以及来自工程文档的原文引用。"),
|
||
("roadmap", "10", "已解决的与下一步", "STATUS & NEXT", "工程",
|
||
"已解决的与下一步 — Natural Memory v2",
|
||
"在 12 GiB GPU 上已经跑通的闭环，以及按对正确率最有帮助的顺序排出的下一阶段任务。"),
|
||
("reproduce", "11", "复现实验", "REPRODUCE", "工程",
|
||
"复现实验 — Natural Memory v2",
|
||
"正式脏数据迁移测试的完整命令、协议参数与产物路径。"),
|
||
]
|
||
|
||
AUTHOR_SITE = "https://wpyw.site"
|
||
AUTHOR_MAIL = "[email protected]"
|
||
|
||
# 每页的头号数字。文档页的页眉由一个**真实数字**领起，而不是只有标题：
|
||
# 这些页面本来就是「证据页」，让它自己先把最强的那条数字说出来。
|
||
# 顺带承担移动端的字阶落差 —— 中文标题在 320px 上推不到 7×，数字可以。
|
||
KEYS = {
|
||
"division": ("441.19", "tok", "批次 B 平均输入前缀 token"),
|
||
"mechanism": ("1,048,576", "", "地址空间设计容量"),
|
||
"results": ("82.52", "%", "批次 B 可回答正确率 · 批次 A 为 62.60%，两批配置不同、不可合并"),
|
||
"nm21": ("94.14", "%", "路由器 Top-1（v2 为 41.12%）"),
|
||
"examples": ("5", "", "逐字原文，其中 2 条是我们主动放进去的判错样例"),
|
||
"ledger": ("108", "", "条真实评测 episode，含全部失败"),
|
||
"cost": ("26.35", "ms", "读取路径额外耗时（批次 B；批次 A 为 61.10 ms）"),
|
||
"capacity": ("512", "B", "每条记录地址字节 · 100 万条约 512 MB"),
|
||
"limits": ("75.00", "%", "跨域未知泄漏率 —— 仍未解决，覆盖门已回退"),
|
||
"roadmap": ("52", "", "项单元测试通过"),
|
||
"reproduce": ("264", "", "题题集 · 246 可回答 / 18 未知"),
|
||
}
|
||
|
||
GROUP_ORDER = ["概览", "原理", "证据", "工程"]
|
||
RELEASE_SLUG = "index" # 这一页用发布页版式，其余用文档版式
|
||
|
||
|
||
def asset_version() -> str:
|
||
"""assets 静态资源的短指纹：内容一变 URL 就变，浏览器不会再用到旧资源。"""
|
||
h = hashlib.md5()
|
||
files = []
|
||
for sub in ("css", "js"):
|
||
d = os.path.join(ROOT, "assets", sub)
|
||
if os.path.isdir(d):
|
||
files += [os.path.join(d, f) for f in sorted(os.listdir(d)) if f.endswith((".css", ".js"))]
|
||
for p in files:
|
||
with open(p, "rb") as fh:
|
||
h.update(fh.read())
|
||
return h.hexdigest()[:10]
|
||
|
||
|
||
def expand_includes(text: str) -> str:
|
||
"""把 {{include:partials/x.svg}} 展开成 src/partials/x.svg 的内容。"""
|
||
def repl(m):
|
||
rel = m.group(1).strip().lstrip("/")
|
||
p = os.path.join(ROOT, "src", rel)
|
||
if not os.path.isfile(p):
|
||
sys.stderr.write("[警告] 找不到 include: %s\n" % rel)
|
||
return "<!-- include missing: %s -->" % rel
|
||
with open(p, "r", encoding="utf-8") as fh:
|
||
return fh.read().rstrip()
|
||
return re.sub(r"\{\{include:([^}]+)\}\}", repl, text)
|
||
|
||
|
||
def href_for(slug: str) -> str:
|
||
return "./" if slug == RELEASE_SLUG else "./%s" % slug
|
||
|
||
|
||
def render_sidebar(active: str) -> str:
|
||
out = ['<a class="side__back" href="./">← 返回发布页</a>']
|
||
for group in GROUP_ORDER:
|
||
items = [p for p in PAGES if p[4] == group and p[0] != RELEASE_SLUG]
|
||
if not items:
|
||
continue
|
||
out.append('<p class="side__grp">%s</p>' % group)
|
||
out.append('<ul class="side__list">')
|
||
for slug, num, zh, en, _g, _t, _d in items:
|
||
cls = "side__a is-on" if slug == active else "side__a"
|
||
cur = ' aria-current="page"' if slug == active else ""
|
||
out.append('<li><a class="%s" href="%s"%s><span class="side__n">%s</span>'
|
||
'<span class="side__t">%s</span></a></li>' % (cls, href_for(slug), cur, num, zh))
|
||
out.append("</ul>")
|
||
return "\n".join(out)
|
||
|
||
|
||
def render_prevnext(active: str) -> str:
|
||
"""上一节 / 下一节：只在文档页出现，且把发布页作为首尾的邻页。"""
|
||
if active == RELEASE_SLUG:
|
||
return ""
|
||
order = [p[0] for p in PAGES]
|
||
idx = order.index(active)
|
||
prev_p = PAGES[idx - 1] if idx > 0 else None
|
||
next_p = PAGES[idx + 1] if idx < len(PAGES) - 1 else None
|
||
|
||
def link(p, direction):
|
||
if not p:
|
||
return '<span class="pn__x"></span>'
|
||
slug, num, zh = p[0], p[1], p[2]
|
||
label = "上一节" if direction == "prev" else "下一节"
|
||
kbd = "←" if direction == "prev" else "→"
|
||
return ('<a class="pn__a pn__a--%s" href="%s" data-dir="%s">'
|
||
'<span class="pn__k">%s · %s</span><span class="pn__t">%s</span>'
|
||
'<span class="kbd">%s</span></a>' % (direction, href_for(slug), direction, label, num, zh, kbd))
|
||
|
||
return '<nav class="pn" aria-label="章节切换">%s%s</nav>' % (link(prev_p, "prev"), link(next_p, "next"))
|
||
|
||
|
||
def render_topnav(active: str) -> str:
|
||
"""顶栏快速导航：发布页与文档页共用，当前页高亮。"""
|
||
items = [
|
||
("index", "发布页"),
|
||
("mechanism", "机制"),
|
||
("results", "测量"),
|
||
("nm21", "NM2.1"),
|
||
("ledger", "台账"),
|
||
("limits", "边界"),
|
||
("reproduce", "复现"),
|
||
]
|
||
out = []
|
||
for slug, label in items:
|
||
cls = "topnav__a is-on" if slug == active else "topnav__a"
|
||
cur = ' aria-current="page"' if slug == active else ""
|
||
out.append('<a class="%s" href="%s"%s>%s</a>' % (cls, href_for(slug), cur, label))
|
||
return " ".join(out)
|
||
|
||
|
||
def render_main(slug: str, num: str, zh: str, en: str, desc: str, content: str) -> str:
|
||
if slug == RELEASE_SLUG:
|
||
# 发布页：无侧栏，全宽分段
|
||
return ('<main class="release" id="doc">\n%s\n</main>' % content)
|
||
fig = ""
|
||
if slug in KEYS:
|
||
k, unit, note = KEYS[slug]
|
||
fig = (' <p class="doc__figure">\n'
|
||
' <span class="mega">%s<small>%s</small></span>\n'
|
||
' <span class="doc__figure-n">%s</span>\n'
|
||
' </p>\n' % (k, unit, note))
|
||
return (
|
||
'<div class="shell">\n'
|
||
' <aside class="side">\n'
|
||
' <nav class="side__in" aria-label="章节导航">\n%s\n </nav>\n'
|
||
' </aside>\n'
|
||
' <main class="doc" id="doc">\n'
|
||
' <header class="doc__hd">\n'
|
||
' <span class="kicker">%s / %s</span>\n'
|
||
' <h1 class="doc__h">%s</h1>\n'
|
||
' <p class="doc__lede">%s</p>\n%s'
|
||
' </header>\n%s\n%s\n </main>\n'
|
||
'</div>' % (render_sidebar(slug), num, en, zh, desc, fig, content, render_prevnext(slug))
|
||
)
|
||
|
||
|
||
_SVG_RE = re.compile(r"<svg\b.*?</svg>", re.S | re.I)
|
||
_TAG_RE = re.compile(r"<[^>]+>")
|
||
_WS_RE = re.compile(r"\s+")
|
||
_ENT = (("<", "<"), (">", ">"), ("&", "&"), (""", '"'),
|
||
("'", "'"), ("←", "←"), ("→", "→"), (" ", " "))
|
||
|
||
|
||
def _plain(html: str) -> str:
|
||
html = _SVG_RE.sub(" ", html)
|
||
html = _TAG_RE.sub(" ", html)
|
||
for a, b in _ENT:
|
||
html = html.replace(a, b)
|
||
return _WS_RE.sub(" ", html).strip()
|
||
|
||
|
||
def build_search_index() -> int:
|
||
"""把每页正文按 h2/h3 切成条目，生成顶栏命令面板用的客户端检索索引。
|
||
|
||
在 asset_version() 之前调用，这样 search.js 的内容也会进资源指纹。
|
||
"""
|
||
entries = []
|
||
for slug, num, zh, en, group, title, desc in PAGES:
|
||
body_path = os.path.join(SRC, "%s.body.html" % slug)
|
||
if not os.path.isfile(body_path):
|
||
continue
|
||
with open(body_path, "r", encoding="utf-8") as fh:
|
||
body = expand_includes(fh.read())
|
||
|
||
chunks = []
|
||
# 以 h2/h3 为锚点切段：parts = [前导, level, 标题, 正文, level, 标题, 正文, ...]
|
||
parts = re.split(r"<h([23])\b[^>]*>(.*?)</h\1>", body, flags=re.S | re.I)
|
||
if len(parts) > 1:
|
||
lead = _plain(parts[0])
|
||
if lead:
|
||
chunks.append((zh, lead))
|
||
for i in range(1, len(parts) - 1, 3):
|
||
head = _plain(parts[i + 1]) or zh
|
||
rest = _plain(parts[i + 2]) if i + 2 < len(parts) else ""
|
||
chunks.append((head, rest))
|
||
else:
|
||
chunks.append((zh, _plain(body)))
|
||
|
||
entries.append({"s": slug, "p": href_for(slug), "g": group, "n": num,
|
||
"pg": zh, "h": "", "t": desc, "d": 1})
|
||
for head, text in chunks:
|
||
entries.append({"s": slug, "p": href_for(slug), "g": group, "n": num,
|
||
"pg": zh, "h": head, "t": text[:220]})
|
||
|
||
out = os.path.join(ROOT, "assets", "js", "search.js")
|
||
with open(out, "w", encoding="utf-8", newline="\n") as fh:
|
||
fh.write("/* 由 build.py 生成：顶栏命令面板的全站检索索引。请勿手改。 */\n")
|
||
fh.write("window.NM_SEARCH = %s;\n"
|
||
% json.dumps(entries, ensure_ascii=False, separators=(",", ":")))
|
||
return len(entries)
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--check", action="store_true", help="只校验，不写文件")
|
||
args = ap.parse_args()
|
||
|
||
if not os.path.isfile(TPL):
|
||
sys.stderr.write("缺少模板 %s\n" % TPL)
|
||
return 2
|
||
with open(TPL, "r", encoding="utf-8") as fh:
|
||
base = fh.read()
|
||
|
||
n_idx = 0 if args.check else build_search_index()
|
||
av = asset_version()
|
||
missing, written = [], []
|
||
for slug, num, zh, en, group, title, desc in PAGES:
|
||
body_path = os.path.join(SRC, "%s.body.html" % slug)
|
||
if not os.path.isfile(body_path):
|
||
missing.append("src/pages/%s.body.html" % slug)
|
||
continue
|
||
with open(body_path, "r", encoding="utf-8") as fh:
|
||
body = expand_includes(fh.read()).rstrip() + "\n"
|
||
|
||
layout = "release" if slug == RELEASE_SLUG else "docs"
|
||
html = (base
|
||
.replace("{{TITLE}}", title)
|
||
.replace("{{DESC}}", desc)
|
||
.replace("{{PAGE_ID}}", slug)
|
||
.replace("{{LAYOUT}}", layout)
|
||
.replace("{{NUM}}", num)
|
||
.replace("{{ZH}}", zh)
|
||
.replace("{{EN}}", en)
|
||
.replace("{{GROUP}}", group)
|
||
.replace("{{ASSET_V}}", av)
|
||
.replace("{{TOPBAR_NAV}}", render_topnav(slug))
|
||
.replace("{{MAIN}}", render_main(slug, num, zh, en, desc, body))
|
||
.replace("{{BUILT}}", datetime.now().strftime("%Y-%m-%d %H:%M")))
|
||
left = re.findall(r"\{\{[A-Z_]+\}\}", html)
|
||
if left:
|
||
sys.stderr.write("[警告] %s 仍有未替换占位符: %s\n" % (slug, set(left)))
|
||
|
||
if not args.check:
|
||
out = os.path.join(ROOT, "index.html" if slug == RELEASE_SLUG else "%s.html" % slug)
|
||
with open(out, "w", encoding="utf-8", newline="\n") as fh:
|
||
fh.write(html)
|
||
written.append("index.html" if slug == RELEASE_SLUG else "%s.html" % slug)
|
||
|
||
if missing:
|
||
sys.stderr.write("[缺失正文片段]\n " + "\n ".join(missing) + "\n")
|
||
print("页面 %d 个%s：" % (len(written), "（校验模式，未写盘）" if args.check else ""))
|
||
for w in written:
|
||
print(" " + w)
|
||
if n_idx:
|
||
print("检索索引 %d 条 → assets/js/search.js" % n_idx)
|
||
return 1 if missing else 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|