Initial commit: 让多个 AI 智能体通过文件总线在共享聊天室里互相 @、协作与交接任务
This commit is contained in:
WpyQwq
@@ -0,0 +1,189 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* model-shootout.mjs — 给 opencode 的免费模型跑同一套可判分题，挑出免费档里最强的那个。
|
||||
*
|
||||
* 用法：
|
||||
* node model-shootout.mjs # 跑默认题组 × 全部免费模型
|
||||
* node model-shootout.mjs --models a,b # 只跑指定模型
|
||||
* node model-shootout.mjs --timeout 240 # 每个模型超时（秒）
|
||||
*
|
||||
* 判分是客观字符串/数值匹配，不打印象分。结果同时打印成表并写入 shootout-result.json。
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const DEFAULT_MODELS = [
|
||||
'opencode/big-pickle',
|
||||
'opencode/muse-spark-1.3-contributor-free',
|
||||
'opencode/muse-spark-1.2-contributor-free',
|
||||
'opencode/nemotron-3-ultra-free',
|
||||
'opencode/nemotron-3.5-lightning-free',
|
||||
'opencode/ling-3.0-flash-fin-free',
|
||||
'opencode/mimo-v2.5-free',
|
||||
];
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
const getFlag = (name, def) => {
|
||||
const i = argv.indexOf(`--${name}`);
|
||||
if (i === -1) return def;
|
||||
const v = argv[i + 1];
|
||||
return v === undefined || v.startsWith('--') ? true : v;
|
||||
};
|
||||
const models = String(getFlag('models', '')).trim()
|
||||
? String(getFlag('models')).split(',').map((s) => s.trim()).filter(Boolean)
|
||||
: DEFAULT_MODELS;
|
||||
const timeoutSec = Number(getFlag('timeout', 300));
|
||||
|
||||
/* --------------------------------------------------------------- 题组 */
|
||||
|
||||
// 长文本里埋一个只出现一次的事实，考长上下文里的定位能力。
|
||||
const HAYSTACK = (() => {
|
||||
const facts = [
|
||||
'仓库代号是 KESTREL-7。',
|
||||
'构建编号是 4412。',
|
||||
'负责人是林工。',
|
||||
'发布窗口在周四凌晨。',
|
||||
'数据库用的是 SQLite。',
|
||||
];
|
||||
const filler = [
|
||||
'这条记录只用于填充上下文，不包含任何需要记忆的信息。',
|
||||
'例行巡检没有发现异常，指标都在阈值内。',
|
||||
'文档结构保持不变，仅调整了少量措辞。',
|
||||
'监控面板显示过去一小时流量平稳。',
|
||||
'值班同学已交接，交接内容无特殊事项。',
|
||||
];
|
||||
const lines = [];
|
||||
for (let i = 0; i < 60; i++) {
|
||||
lines.push(filler[i % filler.length]);
|
||||
if (i === 37) lines.push(`【关键】本段的唯一标识是 ${facts[1]} 备用代号 ${facts[0]}`);
|
||||
}
|
||||
return lines.join('');
|
||||
})();
|
||||
|
||||
const TASKS = [
|
||||
{
|
||||
id: 'Q1',
|
||||
q: '球拍和球一共 1.10 元，球拍比球贵 1.00 元。球多少钱？',
|
||||
// 接受 0.05 / .05 / 五分 / 5分 —— 中文作答也算对（第一版只认 0.05，把「五分」误判成错）
|
||||
grade: (t) => (/(^|[^\d.])0?\.05([^\d]|$)/.test(t) || /[五5]\s*分/.test(t) ? 1 : 0),
|
||||
expect: '0.05（或「五分」/「5分」）',
|
||||
},
|
||||
{
|
||||
id: 'Q2',
|
||||
q: '小于 1000 的正整数里，有多少个满足「除以 7 余 3」？',
|
||||
grade: (t) => (/(^|[^\d])143([^\d]|$)/.test(t) ? 1 : 0),
|
||||
expect: '143',
|
||||
},
|
||||
{
|
||||
id: 'Q3',
|
||||
q: 'Python 表达式 len(list(range(10))[2:5]) 的值是多少？',
|
||||
grade: (t) => (/(^|[^\d])3([^\d]|$)/.test(t) ? 1 : 0),
|
||||
expect: '3',
|
||||
},
|
||||
{
|
||||
id: 'Q4',
|
||||
q: '已知「所有 A 都是 B」且「有些 B 是 C」。能否必然推出「有些 A 是 C」？只回答 能 或 不能。',
|
||||
grade: (t) => (/不能/.test(t) && !/能[^不]/.test(t.replace(/不能/g, '')) ? 1 : 0),
|
||||
expect: '不能',
|
||||
},
|
||||
{
|
||||
id: 'Q5',
|
||||
q: '下面这段文本里出现过一次「构建编号」，它的数字是多少？\n\n' + HAYSTACK,
|
||||
grade: (t) => (/(^|[^\d])4412([^\d]|$)/.test(t) ? 1 : 0),
|
||||
expect: '4412',
|
||||
},
|
||||
];
|
||||
|
||||
const PROMPT = [
|
||||
'按顺序回答下面的问题。严格要求输出格式：每题一行，形如 `Q1) 答案`，',
|
||||
'答案要极短（数字或两个字），不要解释、不要复述题目、不要用 markdown 代码块。',
|
||||
'',
|
||||
...TASKS.map((t) => `${t.id}. ${t.q}`),
|
||||
].join('\n');
|
||||
|
||||
/* --------------------------------------------------------------- 运行 */
|
||||
|
||||
function opencodeExe() {
|
||||
const cands = [
|
||||
path.join(process.env.APPDATA || '', 'npm', 'node_modules', 'opencode-ai', 'bin', 'opencode.exe'),
|
||||
path.join(os.homedir(), '.opencode', 'bin', 'opencode'),
|
||||
];
|
||||
for (const c of cands) if (c && fs.existsSync(c)) return c;
|
||||
return 'opencode';
|
||||
}
|
||||
|
||||
function extract(text, id) {
|
||||
const re = new RegExp(`${id}\\s*[).:：]\\s*([^\\n]+)`, 'i');
|
||||
const m = text.match(re);
|
||||
return m ? m[1].trim() : text;
|
||||
}
|
||||
|
||||
function runModel(model) {
|
||||
const started = Date.now();
|
||||
const r = spawnSync(opencodeExe(), ['run', '-m', model, PROMPT], {
|
||||
encoding: 'utf8',
|
||||
windowsHide: true,
|
||||
timeout: timeoutSec * 1000,
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
cwd: os.tmpdir(),
|
||||
env: { ...process.env, NO_COLOR: '1', AIGROUP_NO_WAKE: '1' },
|
||||
});
|
||||
const ms = Date.now() - started;
|
||||
const stdout = (r.stdout || '').replace(/\u001b\[[0-9;]*m/g, '').trim();
|
||||
const stderr = (r.stderr || '').trim();
|
||||
if (r.error || r.status !== 0) {
|
||||
return { model, ok: false, ms, error: r.error?.message || `exit ${r.status}`, stderr: stderr.slice(-400), raw: stdout.slice(0, 400) };
|
||||
}
|
||||
const per = {};
|
||||
let score = 0;
|
||||
for (const t of TASKS) {
|
||||
const ans = extract(stdout, t.id);
|
||||
const g = t.grade(ans) || t.grade(stdout);
|
||||
per[t.id] = { answer: ans.slice(0, 60), expect: t.expect, pass: Boolean(g) };
|
||||
if (g) score++;
|
||||
}
|
||||
return { model, ok: true, ms, score, total: TASKS.length, per, raw: stdout.slice(0, 1200) };
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------- main */
|
||||
|
||||
const results = [];
|
||||
for (const model of models) {
|
||||
process.stderr.write(`… 跑 ${model}\n`);
|
||||
const res = runModel(model);
|
||||
results.push(res);
|
||||
process.stderr.write(` → ${res.ok ? `${res.score}/${res.total} (${(res.ms / 1000).toFixed(1)}s)` : `失败 ${res.error}`}\n`);
|
||||
}
|
||||
|
||||
results.sort((a, b) => (b.score ?? -1) - (a.score ?? -1) || (a.ms ?? 1e9) - (b.ms ?? 1e9));
|
||||
|
||||
const report = {
|
||||
at: new Date().toISOString(),
|
||||
tasks: TASKS.map((t) => ({ id: t.id, expect: t.expect, q: t.q.slice(0, 80) })),
|
||||
results,
|
||||
};
|
||||
|
||||
fs.writeFileSync(path.join(HERE, 'shootout-result.json'), JSON.stringify(report, null, 2), 'utf8');
|
||||
|
||||
const head = ['排名', '模型', '得分', '耗时', ...TASKS.map((t) => t.id)];
|
||||
const rows = results.map((r, i) => [
|
||||
String(i + 1),
|
||||
r.model.replace('opencode/', ''),
|
||||
r.ok ? `${r.score}/${r.total}` : 'ERR',
|
||||
r.ok ? `${(r.ms / 1000).toFixed(1)}s` : '-',
|
||||
...TASKS.map((t) => (r.ok ? (r.per[t.id].pass ? '✔' : `✘(${r.per[t.id].answer})`) : '-')),
|
||||
]);
|
||||
const widths = head.map((h, i) => Math.max(h.length, ...rows.map((r) => String(r[i] ?? '').length)));
|
||||
const line = (cells) => cells.map((c, i) => String(c ?? '').padEnd(widths[i])).join(' ');
|
||||
|
||||
process.stdout.write(`\n${line(head)}\n${line(widths.map((w) => '─'.repeat(w)))}\n`);
|
||||
for (const r of rows) process.stdout.write(`${line(r)}\n`);
|
||||
const winner = results.find((r) => r.ok);
|
||||
process.stdout.write(`\n最强免费档（本次同题实测）：${winner ? winner.model : '(全部失败)'}\n`);
|
||||
process.stdout.write(`结果已写入 ${path.join(HERE, 'shootout-result.json')}\n`);
|
||||
Reference in New Issue
Block a user