190 lines
7.0 KiB
JavaScript
190 lines
7.0 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* model-shootout.mjs — 给 opencode 的免费模型跑同一套可判分题，挑出免费档里最强的那个。
|
||
*
|
||
* 用法：
|
||
* node model-shootout.mjs # 跑默认题组 × 全部免费模型
|
||
* node model-shootout.mjs --models a,b # 只跑指定模型
|
||
* node model-shootout.mjs --timeout 240 # 每个模型超时（秒）
|
||
*
|
||
* 判分是客观字符串/数值匹配，不打印象分。结果同时打印成表并写入 shootout-result.json。
|
||
*/
|
||
|
||
import fs from 'node:fs';
|
||
import path from 'node:path';
|
||
import os from 'node:os';
|
||
import { spawnSync } from 'node:child_process';
|
||
import { fileURLToPath } from 'node:url';
|
||
|
||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||
|
||
const DEFAULT_MODELS = [
|
||
'opencode/big-pickle',
|
||
'opencode/muse-spark-1.3-contributor-free',
|
||
'opencode/muse-spark-1.2-contributor-free',
|
||
'opencode/nemotron-3-ultra-free',
|
||
'opencode/nemotron-3.5-lightning-free',
|
||
'opencode/ling-3.0-flash-fin-free',
|
||
'opencode/mimo-v2.5-free',
|
||
];
|
||
|
||
const argv = process.argv.slice(2);
|
||
const getFlag = (name, def) => {
|
||
const i = argv.indexOf(`--${name}`);
|
||
if (i === -1) return def;
|
||
const v = argv[i + 1];
|
||
return v === undefined || v.startsWith('--') ? true : v;
|
||
};
|
||
const models = String(getFlag('models', '')).trim()
|
||
? String(getFlag('models')).split(',').map((s) => s.trim()).filter(Boolean)
|
||
: DEFAULT_MODELS;
|
||
const timeoutSec = Number(getFlag('timeout', 300));
|
||
|
||
/* --------------------------------------------------------------- 题组 */
|
||
|
||
// 长文本里埋一个只出现一次的事实，考长上下文里的定位能力。
|
||
const HAYSTACK = (() => {
|
||
const facts = [
|
||
'仓库代号是 KESTREL-7。',
|
||
'构建编号是 4412。',
|
||
'负责人是林工。',
|
||
'发布窗口在周四凌晨。',
|
||
'数据库用的是 SQLite。',
|
||
];
|
||
const filler = [
|
||
'这条记录只用于填充上下文，不包含任何需要记忆的信息。',
|
||
'例行巡检没有发现异常，指标都在阈值内。',
|
||
'文档结构保持不变，仅调整了少量措辞。',
|
||
'监控面板显示过去一小时流量平稳。',
|
||
'值班同学已交接，交接内容无特殊事项。',
|
||
];
|
||
const lines = [];
|
||
for (let i = 0; i < 60; i++) {
|
||
lines.push(filler[i % filler.length]);
|
||
if (i === 37) lines.push(`【关键】本段的唯一标识是 ${facts[1]} 备用代号 ${facts[0]}`);
|
||
}
|
||
return lines.join('');
|
||
})();
|
||
|
||
const TASKS = [
|
||
{
|
||
id: 'Q1',
|
||
q: '球拍和球一共 1.10 元，球拍比球贵 1.00 元。球多少钱？',
|
||
// 接受 0.05 / .05 / 五分 / 5分 —— 中文作答也算对（第一版只认 0.05，把「五分」误判成错）
|
||
grade: (t) => (/(^|[^\d.])0?\.05([^\d]|$)/.test(t) || /[五5]\s*分/.test(t) ? 1 : 0),
|
||
expect: '0.05（或「五分」/「5分」）',
|
||
},
|
||
{
|
||
id: 'Q2',
|
||
q: '小于 1000 的正整数里，有多少个满足「除以 7 余 3」？',
|
||
grade: (t) => (/(^|[^\d])143([^\d]|$)/.test(t) ? 1 : 0),
|
||
expect: '143',
|
||
},
|
||
{
|
||
id: 'Q3',
|
||
q: 'Python 表达式 len(list(range(10))[2:5]) 的值是多少？',
|
||
grade: (t) => (/(^|[^\d])3([^\d]|$)/.test(t) ? 1 : 0),
|
||
expect: '3',
|
||
},
|
||
{
|
||
id: 'Q4',
|
||
q: '已知「所有 A 都是 B」且「有些 B 是 C」。能否必然推出「有些 A 是 C」？只回答 能 或 不能。',
|
||
grade: (t) => (/不能/.test(t) && !/能[^不]/.test(t.replace(/不能/g, '')) ? 1 : 0),
|
||
expect: '不能',
|
||
},
|
||
{
|
||
id: 'Q5',
|
||
q: '下面这段文本里出现过一次「构建编号」，它的数字是多少？\n\n' + HAYSTACK,
|
||
grade: (t) => (/(^|[^\d])4412([^\d]|$)/.test(t) ? 1 : 0),
|
||
expect: '4412',
|
||
},
|
||
];
|
||
|
||
const PROMPT = [
|
||
'按顺序回答下面的问题。严格要求输出格式：每题一行，形如 `Q1) 答案`，',
|
||
'答案要极短（数字或两个字），不要解释、不要复述题目、不要用 markdown 代码块。',
|
||
'',
|
||
...TASKS.map((t) => `${t.id}. ${t.q}`),
|
||
].join('\n');
|
||
|
||
/* --------------------------------------------------------------- 运行 */
|
||
|
||
function opencodeExe() {
|
||
const cands = [
|
||
path.join(process.env.APPDATA || '', 'npm', 'node_modules', 'opencode-ai', 'bin', 'opencode.exe'),
|
||
path.join(os.homedir(), '.opencode', 'bin', 'opencode'),
|
||
];
|
||
for (const c of cands) if (c && fs.existsSync(c)) return c;
|
||
return 'opencode';
|
||
}
|
||
|
||
function extract(text, id) {
|
||
const re = new RegExp(`${id}\\s*[).:：]\\s*([^\\n]+)`, 'i');
|
||
const m = text.match(re);
|
||
return m ? m[1].trim() : text;
|
||
}
|
||
|
||
function runModel(model) {
|
||
const started = Date.now();
|
||
const r = spawnSync(opencodeExe(), ['run', '-m', model, PROMPT], {
|
||
encoding: 'utf8',
|
||
windowsHide: true,
|
||
timeout: timeoutSec * 1000,
|
||
maxBuffer: 16 * 1024 * 1024,
|
||
cwd: os.tmpdir(),
|
||
env: { ...process.env, NO_COLOR: '1', AIGROUP_NO_WAKE: '1' },
|
||
});
|
||
const ms = Date.now() - started;
|
||
const stdout = (r.stdout || '').replace(/\u001b\[[0-9;]*m/g, '').trim();
|
||
const stderr = (r.stderr || '').trim();
|
||
if (r.error || r.status !== 0) {
|
||
return { model, ok: false, ms, error: r.error?.message || `exit ${r.status}`, stderr: stderr.slice(-400), raw: stdout.slice(0, 400) };
|
||
}
|
||
const per = {};
|
||
let score = 0;
|
||
for (const t of TASKS) {
|
||
const ans = extract(stdout, t.id);
|
||
const g = t.grade(ans) || t.grade(stdout);
|
||
per[t.id] = { answer: ans.slice(0, 60), expect: t.expect, pass: Boolean(g) };
|
||
if (g) score++;
|
||
}
|
||
return { model, ok: true, ms, score, total: TASKS.length, per, raw: stdout.slice(0, 1200) };
|
||
}
|
||
|
||
/* --------------------------------------------------------------- main */
|
||
|
||
const results = [];
|
||
for (const model of models) {
|
||
process.stderr.write(`… 跑 ${model}\n`);
|
||
const res = runModel(model);
|
||
results.push(res);
|
||
process.stderr.write(` → ${res.ok ? `${res.score}/${res.total} (${(res.ms / 1000).toFixed(1)}s)` : `失败 ${res.error}`}\n`);
|
||
}
|
||
|
||
results.sort((a, b) => (b.score ?? -1) - (a.score ?? -1) || (a.ms ?? 1e9) - (b.ms ?? 1e9));
|
||
|
||
const report = {
|
||
at: new Date().toISOString(),
|
||
tasks: TASKS.map((t) => ({ id: t.id, expect: t.expect, q: t.q.slice(0, 80) })),
|
||
results,
|
||
};
|
||
|
||
fs.writeFileSync(path.join(HERE, 'shootout-result.json'), JSON.stringify(report, null, 2), 'utf8');
|
||
|
||
const head = ['排名', '模型', '得分', '耗时', ...TASKS.map((t) => t.id)];
|
||
const rows = results.map((r, i) => [
|
||
String(i + 1),
|
||
r.model.replace('opencode/', ''),
|
||
r.ok ? `${r.score}/${r.total}` : 'ERR',
|
||
r.ok ? `${(r.ms / 1000).toFixed(1)}s` : '-',
|
||
...TASKS.map((t) => (r.ok ? (r.per[t.id].pass ? '✔' : `✘(${r.per[t.id].answer})`) : '-')),
|
||
]);
|
||
const widths = head.map((h, i) => Math.max(h.length, ...rows.map((r) => String(r[i] ?? '').length)));
|
||
const line = (cells) => cells.map((c, i) => String(c ?? '').padEnd(widths[i])).join(' ');
|
||
|
||
process.stdout.write(`\n${line(head)}\n${line(widths.map((w) => '─'.repeat(w)))}\n`);
|
||
for (const r of rows) process.stdout.write(`${line(r)}\n`);
|
||
const winner = results.find((r) => r.ok);
|
||
process.stdout.write(`\n最强免费档（本次同题实测）：${winner ? winner.model : '(全部失败)'}\n`);
|
||
process.stdout.write(`结果已写入 ${path.join(HERE, 'shootout-result.json')}\n`);
|