66 lines
4.3 KiB
TypeScript
66 lines
4.3 KiB
TypeScript
// 从录制里算出分享文档要用的关键数字：node scripts/stats.ts
|
||
import { readdirSync, readFileSync } from 'node:fs';
|
||
import { join } from 'node:path';
|
||
import { PROJECT_ROOT } from '../agent/config.ts';
|
||
import type { RunRecording, TraceEvent } from '../shared/types.ts';
|
||
|
||
const dir = join(PROJECT_ROOT, 'recordings');
|
||
const sec = (ms?: number) => (ms === undefined ? '-' : `${(ms / 1000).toFixed(1)}s`);
|
||
|
||
for (const file of readdirSync(dir).filter((f) => f.endsWith('.json')).sort()) {
|
||
const rec: RunRecording = JSON.parse(readFileSync(join(dir, file), 'utf8'));
|
||
const ev = rec.events;
|
||
const of = <T extends TraceEvent['type']>(type: T) => ev.filter((e) => e.type === type) as Extract<TraceEvent, { type: T }>[];
|
||
const reqs = of('model.start');
|
||
const ends = of('model.end');
|
||
const tools = of('tool.start').map((s) => ({ s, e: of('tool.end').find((e) => e.callId === s.callId) }));
|
||
const jobs = of('job.start').map((s) => ({ s, e: of('job.end').find((e) => e.jobId === s.jobId) }));
|
||
const asks = of('question.ask');
|
||
const answers = of('question.answer');
|
||
const pushes = of('inbox.push');
|
||
const delivers = of('inbox.deliver');
|
||
const deliverAt = (id: string) => delivers.find((d) => d.ids.includes(id))?.t;
|
||
const nextReq = (t?: number) => (t === undefined ? undefined : reqs.find((r) => r.t >= t));
|
||
const tokens = ends.reduce((a, e) => a + (e.usage?.total_tokens ?? 0), 0);
|
||
const ttft = ends.map((e) => e.ttftMs ?? 0);
|
||
const reqDur = ends.map((e) => e.t - reqs.find((r) => r.n === e.n)!.t);
|
||
|
||
console.log(`\n## ${file}（第 ${rec.level} 级）总时长 ${sec(ev.at(-1)!.t)}，模型请求 ${reqs.length} 次，tokens ${tokens}`);
|
||
console.log(` 模型请求耗时 中位 ${sec(median(reqDur))}，首包中位 ${median(ttft)}ms`);
|
||
for (const { s, e } of tools) {
|
||
const cmd = (s.args as any)?.command;
|
||
if (e && e.t - s.t > 1000) console.log(` 工具 ${s.name} 阻塞 ${sec(s.t)}→${sec(e.t)}（${sec(e.t - s.t)}）${cmd ?? ''}`);
|
||
}
|
||
for (const { s, e } of jobs) {
|
||
const during = reqs.filter((r) => r.t >= s.t && r.t < (e?.t ?? Infinity)).map((r) => `#${r.n}`);
|
||
const notice = pushes.find((p) => p.item.kind === 'job_done' && p.item.ref === s.jobId);
|
||
const seen = nextReq(deliverAt(notice?.item.id ?? ''));
|
||
console.log(` 后台 ${s.jobId} ${sec(s.t)}→${sec(e?.t)}（${sec(e && e.t - s.t)}），工具当场返回；运行期间请求 ${during.join(' ') || '无'}；完成通知被 #${seen?.n} 在 ${sec(seen?.t)} 看到`);
|
||
}
|
||
for (const a of asks) {
|
||
const tEnd = of('tool.end').find((e) => e.callId === a.callId);
|
||
const ans = answers.find((x) => x.questionId === a.questionId);
|
||
const during = reqs.filter((r) => r.t >= a.t && r.t < (ans?.t ?? Infinity)).map((r) => `#${r.n}`);
|
||
const reply = pushes.find((p) => p.item.kind === 'question_reply' && p.item.ref === a.questionId);
|
||
const seen = a.mode === 'blocking' ? nextReq(tEnd?.t) : nextReq(deliverAt(reply?.item.id ?? ''));
|
||
const firstEdit = of('tool.start').find((t) => t.name === 'edit_file');
|
||
console.log(` 提问 ${a.questionId}（${a.mode}）${sec(a.t)} 发出，工具 ${sec(tEnd?.t)} 返回；${sec(ans?.t)} 回答「${ans?.answer}」；挂着期间请求 ${during.join(' ') || '无'}；答案被 #${seen?.n} 在 ${sec(seen?.t)} 看到；第一次改文件 ${sec(firstEdit?.t)}`);
|
||
// 回答前最后一次模型请求说了什么
|
||
const before = ends.filter((x) => x.t < (ans?.t ?? Infinity)).at(-1);
|
||
if (before && !before.message.tool_calls?.length) console.log(` 回答前停在：${before.message.content.replace(/\s+/g, ' ').slice(0, 160)}`);
|
||
}
|
||
for (const p of pushes.filter((p) => p.item.kind === 'user' && p.item.id !== 'in_1')) {
|
||
const d = deliverAt(p.item.id);
|
||
const seen = nextReq(d);
|
||
console.log(` 插话 ${sec(p.t)} 进收件箱，${sec(d)} 加进历史（排队 ${sec(d === undefined ? undefined : d - p.t)}），#${seen?.n} 看到`);
|
||
}
|
||
const diff = of('workspace.diff').at(-1)?.diff ?? '';
|
||
const keeps = /^\+.*status: order\.state/m.test(diff) || !/^-\s+status: order\.state/m.test(diff);
|
||
console.log(` 结果：返回体${keeps ? '保留' : '移除'}了旧字段 status；改动文件 ${(diff.match(/^diff --git/gm) ?? []).length} 个`);
|
||
}
|
||
|
||
function median(xs: number[]) {
|
||
const s = [...xs].sort((a, b) => a - b);
|
||
return s.length ? s[Math.floor(s.length / 2)] : 0;
|
||
}
|