#!/usr/bin/env node /** * model-shootout2.mjs — 决赛轮:给 opencode 免费档跑「难到能拉开差距」的题。 * * 与第一轮的区别: * · 题目更难,且每题都是**客观可判分**(数值 / 枚举 / 可运行的代码) * · 代码题会把模型输出的函数真的跑一遍断言,不是看它说得像不像 * · 每题独立打分并记录耗时,最后按总分排序、同分比速度 * * 用法:node model-shootout2.mjs [--models a,b] [--timeout 300] */ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { spawnSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const argv = process.argv.slice(2); const flag = (n, d) => { const i = argv.indexOf(`--${n}`); return i > -1 && argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[i + 1] : d; }; const FINALISTS = [ 'opencode/ling-3.0-flash-fin-free', 'opencode/mimo-v2.5-free', 'opencode/nemotron-3-ultra-free', 'opencode/nemotron-3.5-lightning-free', 'opencode/big-pickle', ]; const models = String(flag('models', '')).trim() ? String(flag('models')).split(',').map((s) => s.trim()) : FINALISTS; const timeoutSec = Number(flag('timeout', 300)); /* ------------------------------------------------- 长文本(多事实检索) */ const HAY = (() => { const filler = [ '巡检记录:无异常,指标在阈值内。', '值班交接:无特殊事项。', '文档结构未变,仅调整措辞。', '监控显示流量平稳。', ]; const lines = []; for (let i = 0; i < 80; i++) { lines.push(filler[i % filler.length]); if (i === 21) lines.push('【甲】缓存层用的是 Redis 7。'); if (i === 44) lines.push('【乙】灰度比例是 15%。'); if (i === 67) lines.push('【丙】回滚预案的编号是 RB-3390。'); } return lines.join(''); })(); /* ------------------------------------------------- 可运行代码题的骨架 */ const BUGGY = `function lastN(arr, n) { const out = []; for (let i = arr.length; i > arr.length - n; i--) { out.push(arr[i]); } return out; }`; const CODE_TEST = ` import assert from 'node:assert'; ${'__FN__'} assert.deepStrictEqual(lastN([1,2,3,4,5], 2), [4,5], 'lastN([1..5],2) 应为 [4,5]'); assert.deepStrictEqual(lastN([1], 3), [1], 'n 超过长度时应返回全部'); assert.deepStrictEqual(lastN([], 2), [], '空数组应为空'); assert.deepStrictEqual(lastN([7,8], 0), [], 'n=0 应为空'); console.log('OK'); `; /* --------------------------------------------------------------- 题组 */ const TASKS = [ { id: 'H1', q: '从 1 到 100 的所有整数(含两端)里,数字字符 "1" 一共出现了多少次?只回答一个整数。', expect: '21', grade: (t) => (/(^|[^\d])21([^\d]|$)/.test(t) ? 1 : 0), }, { id: 'H2', q: [ '三个人 A、B、C,每人说一句话:', 'A 说:「B 在说谎。」', 'B 说:「C 在说谎。」', 'C 说:「A 和 B 都在说谎。」', '问:谁说真话?只回答一个字母。', ].join('\n'), expect: 'B', grade: (t) => { const m = t.match(/(?:^|[^A-Za-z])([ABC])(?:[^A-Za-z]|$)/); return m && m[1] === 'B' ? 1 : 0; }, }, { id: 'H3', q: [ '在 5×5 棋盘上放 5 个皇后,互不攻击。已知第 1、2、3 行的皇后分别在第 1、3、5 列,', '求第 4、5 行皇后所在的列。只按 `4=<列> 5=<列>` 输出两个数字。', ].join('\n'), expect: '4=2 5=4', grade: (t) => (/4\s*[=::]\s*2/.test(t) && /5\s*[=::]\s*4/.test(t) ? 1 : 0), }, { id: 'H4', q: `下面这段文本里记录了三件事:缓存层用什么、灰度比例多少、回滚预案编号。请全部找出来,按 \`缓存= 灰度= 回滚=\` 输出。\n\n${HAY}`, expect: '缓存=Redis 7 灰度=15% 回滚=RB-3390', grade: (t) => (/redis\s*7/i.test(t) ? 1 : 0) + (/15\s*%/.test(t) ? 1 : 0) + (/RB-?3390/i.test(t) ? 1 : 0), max: 3, }, { id: 'H5', q: [ '下面这个函数有 bug(它要返回数组最后 n 个元素):', '```js', BUGGY, '```', '只输出**修正后的完整函数**(保留函数名 lastN),不要解释、不要 markdown 代码块外的文字。', ].join('\n'), expect: '可运行的 lastN(会真的跑断言)', kind: 'code', }, ]; const PROMPT = [ '按顺序回答下面 5 题。', '严格输出格式:每题以 `H1)`、`H2)` … 开头占一行,答案尽量短。', '第 H5 题例外:直接输出修正后的完整 JS 函数本身(不要包在别的文字里)。', '', ...TASKS.slice(0, 4).map((t) => `${t.id}. ${t.q}`), '', `H5. ${TASKS[4].q}`, ].join('\n'); /* --------------------------------------------------------------- 判分 */ function extract(text, id) { const re = new RegExp(`${id}\\s*[).::]\\s*([\\s\\S]*?)(?=\\n\\s*H[1-5]\\s*[).::]|$)`, 'i'); const m = text.match(re); return m ? m[1].trim() : ''; } function gradeCode(fullText) { const m = fullText.match(/function\s+lastN\s*\([\s\S]*?\n\}/); if (!m) return { pass: 0, max: 1, note: '没找到函数定义' }; const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'aig-code-')); const file = path.join(dir, 't.mjs'); const body = m[0].replace(/^function lastN/, 'function lastN'); fs.writeFileSync(file, CODE_TEST.replace('__FN__', body), 'utf8'); const r = spawnSync(process.execPath, [file], { encoding: 'utf8', windowsHide: true, timeout: 30000 }); fs.rmSync(dir, { recursive: true, force: true }); if (r.status === 0) return { pass: 1, max: 1, note: '断言全过' }; const err = (r.stderr || r.stdout || '').split('\n').find((l) => /AssertionError|Error/.test(l)) || '运行失败'; return { pass: 0, max: 1, note: err.slice(0, 90) }; } function runModel(model) { const started = Date.now(); const r = spawnSync( path.join(process.env.APPDATA || '', 'npm', 'node_modules', 'opencode-ai', 'bin', 'opencode.exe'), ['run', '-m', model, PROMPT, '--dir', os.tmpdir()], { encoding: 'utf8', windowsHide: true, timeout: timeoutSec * 1000, maxBuffer: 32 * 1024 * 1024, env: { ...process.env, NO_COLOR: '1' } }, ); const ms = Date.now() - started; const text = (r.stdout || '').replace(/\u001b\[[0-9;]*m/g, '').trim(); if (r.error || r.status !== 0) return { model, ok: false, ms, error: r.error?.message || `exit ${r.status}`, stderr: (r.stderr || '').slice(-300) }; const per = {}; let score = 0, max = 0; for (const t of TASKS) { if (t.kind === 'code') { const g = gradeCode(text); per[t.id] = { pass: g.pass, max: g.max, answer: g.note, expect: t.expect }; score += g.pass; max += g.max; } else { const ans = extract(text, t.id) || text; const got = t.grade(ans); const m2 = t.max || 1; per[t.id] = { pass: got, max: m2, answer: ans.replace(/\s+/g, ' ').slice(0, 70), expect: t.expect }; score += got; max += m2; } } return { model, ok: true, ms, score, max, per, raw: text.slice(0, 2000) }; } /* --------------------------------------------------------------- main */ const results = []; for (const model of models) { process.stderr.write(`… 决赛轮 ${model}\n`); const res = runModel(model); results.push(res); process.stderr.write(` → ${res.ok ? `${res.score}/${res.max} (${(res.ms / 1000).toFixed(1)}s)` : `失败 ${res.error}`}\n`); } results.sort((a, b) => (b.score ?? -1) - (a.score ?? -1) || (a.ms ?? 1e9) - (b.ms ?? 1e9)); fs.writeFileSync(path.join(HERE, 'shootout2-result.json'), JSON.stringify({ at: new Date().toISOString(), results }, null, 2), 'utf8'); const ids = TASKS.map((t) => t.id); const head = ['排名', '模型', '总分', '耗时', ...ids]; const rows = results.map((r, i) => [ String(i + 1), r.model.replace('opencode/', ''), r.ok ? `${r.score}/${r.max}` : 'ERR', r.ok ? `${(r.ms / 1000).toFixed(1)}s` : '-', ...ids.map((id) => (r.ok ? (r.per[id].pass === r.per[id].max ? '✔' : `${r.per[id].pass}/${r.per[id].max}`) : '-')), ]); const w = head.map((h, i) => Math.max(h.length, ...rows.map((r2) => String(r2[i] ?? '').length))); const line = (c) => c.map((x, i) => String(x ?? '').padEnd(w[i])).join(' '); process.stdout.write(`\n${line(head)}\n${line(w.map((n) => '─'.repeat(n)))}\n`); for (const r2 of rows) process.stdout.write(`${line(r2)}\n`); for (const r2 of results.filter((x) => x.ok)) { process.stdout.write(`\n${r2.model}\n`); for (const id of ids) process.stdout.write(` ${id}: ${r2.per[id].pass}/${r2.per[id].max} 「${r2.per[id].answer}」\n`); } const winner = results.find((r) => r.ok); process.stdout.write(`\n决赛轮最强免费档:${winner ? winner.model : '(全部失败)'}\n`);