219 lines
8.6 KiB
JavaScript
219 lines
8.6 KiB
JavaScript
#!/usr/bin/env node
|
||
/**
|
||
* model-shootout2.mjs — 决赛轮：给 opencode 免费档跑「难到能拉开差距」的题。
|
||
*
|
||
* 与第一轮的区别：
|
||
* · 题目更难，且每题都是**客观可判分**（数值 / 枚举 / 可运行的代码）
|
||
* · 代码题会把模型输出的函数真的跑一遍断言，不是看它说得像不像
|
||
* · 每题独立打分并记录耗时，最后按总分排序、同分比速度
|
||
*
|
||
* 用法：node model-shootout2.mjs [--models a,b] [--timeout 300]
|
||
*/
|
||
|
||
import fs from 'node:fs';
|
||
import os from 'node:os';
|
||
import path from 'node:path';
|
||
import { spawnSync } from 'node:child_process';
|
||
import { fileURLToPath } from 'node:url';
|
||
|
||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||
const argv = process.argv.slice(2);
|
||
const flag = (n, d) => { const i = argv.indexOf(`--${n}`); return i > -1 && argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[i + 1] : d; };
|
||
|
||
const FINALISTS = [
|
||
'opencode/ling-3.0-flash-fin-free',
|
||
'opencode/mimo-v2.5-free',
|
||
'opencode/nemotron-3-ultra-free',
|
||
'opencode/nemotron-3.5-lightning-free',
|
||
'opencode/big-pickle',
|
||
];
|
||
const models = String(flag('models', '')).trim() ? String(flag('models')).split(',').map((s) => s.trim()) : FINALISTS;
|
||
const timeoutSec = Number(flag('timeout', 300));
|
||
|
||
/* ------------------------------------------------- 长文本（多事实检索） */
|
||
|
||
const HAY = (() => {
|
||
const filler = [
|
||
'巡检记录：无异常，指标在阈值内。',
|
||
'值班交接：无特殊事项。',
|
||
'文档结构未变，仅调整措辞。',
|
||
'监控显示流量平稳。',
|
||
];
|
||
const lines = [];
|
||
for (let i = 0; i < 80; i++) {
|
||
lines.push(filler[i % filler.length]);
|
||
if (i === 21) lines.push('【甲】缓存层用的是 Redis 7。');
|
||
if (i === 44) lines.push('【乙】灰度比例是 15%。');
|
||
if (i === 67) lines.push('【丙】回滚预案的编号是 RB-3390。');
|
||
}
|
||
return lines.join('');
|
||
})();
|
||
|
||
/* ------------------------------------------------- 可运行代码题的骨架 */
|
||
|
||
const BUGGY = `function lastN(arr, n) {
|
||
const out = [];
|
||
for (let i = arr.length; i > arr.length - n; i--) {
|
||
out.push(arr[i]);
|
||
}
|
||
return out;
|
||
}`;
|
||
|
||
const CODE_TEST = `
|
||
import assert from 'node:assert';
|
||
${'__FN__'}
|
||
assert.deepStrictEqual(lastN([1,2,3,4,5], 2), [4,5], 'lastN([1..5],2) 应为 [4,5]');
|
||
assert.deepStrictEqual(lastN([1], 3), [1], 'n 超过长度时应返回全部');
|
||
assert.deepStrictEqual(lastN([], 2), [], '空数组应为空');
|
||
assert.deepStrictEqual(lastN([7,8], 0), [], 'n=0 应为空');
|
||
console.log('OK');
|
||
`;
|
||
|
||
/* --------------------------------------------------------------- 题组 */
|
||
|
||
const TASKS = [
|
||
{
|
||
id: 'H1',
|
||
q: '从 1 到 100 的所有整数（含两端）里，数字字符 "1" 一共出现了多少次？只回答一个整数。',
|
||
expect: '21',
|
||
grade: (t) => (/(^|[^\d])21([^\d]|$)/.test(t) ? 1 : 0),
|
||
},
|
||
{
|
||
id: 'H2',
|
||
q: [
|
||
'三个人 A、B、C，每人说一句话：',
|
||
'A 说：「B 在说谎。」',
|
||
'B 说：「C 在说谎。」',
|
||
'C 说：「A 和 B 都在说谎。」',
|
||
'问：谁说真话？只回答一个字母。',
|
||
].join('\n'),
|
||
expect: 'B',
|
||
grade: (t) => {
|
||
const m = t.match(/(?:^|[^A-Za-z])([ABC])(?:[^A-Za-z]|$)/);
|
||
return m && m[1] === 'B' ? 1 : 0;
|
||
},
|
||
},
|
||
{
|
||
id: 'H3',
|
||
q: [
|
||
'在 5×5 棋盘上放 5 个皇后，互不攻击。已知第 1、2、3 行的皇后分别在第 1、3、5 列，',
|
||
'求第 4、5 行皇后所在的列。只按 `4=<列> 5=<列>` 输出两个数字。',
|
||
].join('\n'),
|
||
expect: '4=2 5=4',
|
||
grade: (t) => (/4\s*[=:：]\s*2/.test(t) && /5\s*[=:：]\s*4/.test(t) ? 1 : 0),
|
||
},
|
||
{
|
||
id: 'H4',
|
||
q: `下面这段文本里记录了三件事：缓存层用什么、灰度比例多少、回滚预案编号。请全部找出来，按 \`缓存=<x> 灰度=<y> 回滚=<z>\` 输出。\n\n${HAY}`,
|
||
expect: '缓存=Redis 7 灰度=15% 回滚=RB-3390',
|
||
grade: (t) => (/redis\s*7/i.test(t) ? 1 : 0) + (/15\s*%/.test(t) ? 1 : 0) + (/RB-?3390/i.test(t) ? 1 : 0),
|
||
max: 3,
|
||
},
|
||
{
|
||
id: 'H5',
|
||
q: [
|
||
'下面这个函数有 bug（它要返回数组最后 n 个元素）：',
|
||
'```js',
|
||
BUGGY,
|
||
'```',
|
||
'只输出**修正后的完整函数**（保留函数名 lastN），不要解释、不要 markdown 代码块外的文字。',
|
||
].join('\n'),
|
||
expect: '可运行的 lastN（会真的跑断言）',
|
||
kind: 'code',
|
||
},
|
||
];
|
||
|
||
const PROMPT = [
|
||
'按顺序回答下面 5 题。',
|
||
'严格输出格式：每题以 `H1)`、`H2)` … 开头占一行，答案尽量短。',
|
||
'第 H5 题例外：直接输出修正后的完整 JS 函数本身（不要包在别的文字里）。',
|
||
'',
|
||
...TASKS.slice(0, 4).map((t) => `${t.id}. ${t.q}`),
|
||
'',
|
||
`H5. ${TASKS[4].q}`,
|
||
].join('\n');
|
||
|
||
/* --------------------------------------------------------------- 判分 */
|
||
|
||
function extract(text, id) {
|
||
const re = new RegExp(`${id}\\s*[).:：]\\s*([\\s\\S]*?)(?=\\n\\s*H[1-5]\\s*[).:：]|$)`, 'i');
|
||
const m = text.match(re);
|
||
return m ? m[1].trim() : '';
|
||
}
|
||
|
||
function gradeCode(fullText) {
|
||
const m = fullText.match(/function\s+lastN\s*\([\s\S]*?\n\}/);
|
||
if (!m) return { pass: 0, max: 1, note: '没找到函数定义' };
|
||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'aig-code-'));
|
||
const file = path.join(dir, 't.mjs');
|
||
const body = m[0].replace(/^function lastN/, 'function lastN');
|
||
fs.writeFileSync(file, CODE_TEST.replace('__FN__', body), 'utf8');
|
||
const r = spawnSync(process.execPath, [file], { encoding: 'utf8', windowsHide: true, timeout: 30000 });
|
||
fs.rmSync(dir, { recursive: true, force: true });
|
||
if (r.status === 0) return { pass: 1, max: 1, note: '断言全过' };
|
||
const err = (r.stderr || r.stdout || '').split('\n').find((l) => /AssertionError|Error/.test(l)) || '运行失败';
|
||
return { pass: 0, max: 1, note: err.slice(0, 90) };
|
||
}
|
||
|
||
function runModel(model) {
|
||
const started = Date.now();
|
||
const r = spawnSync(
|
||
path.join(process.env.APPDATA || '', 'npm', 'node_modules', 'opencode-ai', 'bin', 'opencode.exe'),
|
||
['run', '-m', model, PROMPT, '--dir', os.tmpdir()],
|
||
{ encoding: 'utf8', windowsHide: true, timeout: timeoutSec * 1000, maxBuffer: 32 * 1024 * 1024, env: { ...process.env, NO_COLOR: '1' } },
|
||
);
|
||
const ms = Date.now() - started;
|
||
const text = (r.stdout || '').replace(/\u001b\[[0-9;]*m/g, '').trim();
|
||
if (r.error || r.status !== 0) return { model, ok: false, ms, error: r.error?.message || `exit ${r.status}`, stderr: (r.stderr || '').slice(-300) };
|
||
|
||
const per = {};
|
||
let score = 0, max = 0;
|
||
for (const t of TASKS) {
|
||
if (t.kind === 'code') {
|
||
const g = gradeCode(text);
|
||
per[t.id] = { pass: g.pass, max: g.max, answer: g.note, expect: t.expect };
|
||
score += g.pass; max += g.max;
|
||
} else {
|
||
const ans = extract(text, t.id) || text;
|
||
const got = t.grade(ans);
|
||
const m2 = t.max || 1;
|
||
per[t.id] = { pass: got, max: m2, answer: ans.replace(/\s+/g, ' ').slice(0, 70), expect: t.expect };
|
||
score += got; max += m2;
|
||
}
|
||
}
|
||
return { model, ok: true, ms, score, max, per, raw: text.slice(0, 2000) };
|
||
}
|
||
|
||
/* --------------------------------------------------------------- main */
|
||
|
||
const results = [];
|
||
for (const model of models) {
|
||
process.stderr.write(`… 决赛轮 ${model}\n`);
|
||
const res = runModel(model);
|
||
results.push(res);
|
||
process.stderr.write(` → ${res.ok ? `${res.score}/${res.max} (${(res.ms / 1000).toFixed(1)}s)` : `失败 ${res.error}`}\n`);
|
||
}
|
||
|
||
results.sort((a, b) => (b.score ?? -1) - (a.score ?? -1) || (a.ms ?? 1e9) - (b.ms ?? 1e9));
|
||
fs.writeFileSync(path.join(HERE, 'shootout2-result.json'), JSON.stringify({ at: new Date().toISOString(), results }, null, 2), 'utf8');
|
||
|
||
const ids = TASKS.map((t) => t.id);
|
||
const head = ['排名', '模型', '总分', '耗时', ...ids];
|
||
const rows = results.map((r, i) => [
|
||
String(i + 1),
|
||
r.model.replace('opencode/', ''),
|
||
r.ok ? `${r.score}/${r.max}` : 'ERR',
|
||
r.ok ? `${(r.ms / 1000).toFixed(1)}s` : '-',
|
||
...ids.map((id) => (r.ok ? (r.per[id].pass === r.per[id].max ? '✔' : `${r.per[id].pass}/${r.per[id].max}`) : '-')),
|
||
]);
|
||
const w = head.map((h, i) => Math.max(h.length, ...rows.map((r2) => String(r2[i] ?? '').length)));
|
||
const line = (c) => c.map((x, i) => String(x ?? '').padEnd(w[i])).join(' ');
|
||
process.stdout.write(`\n${line(head)}\n${line(w.map((n) => '─'.repeat(n)))}\n`);
|
||
for (const r2 of rows) process.stdout.write(`${line(r2)}\n`);
|
||
for (const r2 of results.filter((x) => x.ok)) {
|
||
process.stdout.write(`\n${r2.model}\n`);
|
||
for (const id of ids) process.stdout.write(` ${id}: ${r2.per[id].pass}/${r2.per[id].max} 「${r2.per[id].answer}」\n`);
|
||
}
|
||
const winner = results.find((r) => r.ok);
|
||
process.stdout.write(`\n决赛轮最强免费档：${winner ? winner.model : '(全部失败)'}\n`);
|