first commit
This commit is contained in:
魏天炜
156
async-agent-lab/scripts/drive.ts
Normal file
156
async-agent-lab/scripts/drive.ts
Normal file
@@ -0,0 +1,156 @@
|
||||
// 命令行驱动：对真实模型跑一个级别，打印时间线，把整次运行存进 recordings/。
|
||||
//
|
||||
// node scripts/drive.ts --level 1 [--steer 4000:"顺便说下 status 在哪些文件里用到"]
|
||||
// node scripts/drive.ts --level 4 --answer idle:保留一版 # 等 Agent 停在依赖处再回答
|
||||
// node scripts/drive.ts --level 4 --answer 3000:直接移除 # 提问后 3 秒就回答
|
||||
//
|
||||
// 结束条件：Agent 空闲，并且没有在跑的后台命令、没有待回答的问题。
|
||||
import { mkdirSync, writeFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { parseArgs } from 'node:util';
|
||||
import { loadProvider, PROJECT_ROOT } from '../agent/config.ts';
|
||||
import { createLLM } from '../agent/llm.ts';
|
||||
import { Run } from '../agent/run.ts';
|
||||
import type { TraceEvent } from '../shared/types.ts';
|
||||
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
level: { type: 'string', default: '1' },
|
||||
answer: { type: 'string' }, // idle:选项 或 <毫秒>:选项
|
||||
steer: { type: 'string' }, // <毫秒>:消息，或 test+<毫秒>:消息（第一次 npm test 启动后多少毫秒插话）
|
||||
out: { type: 'string' },
|
||||
'idle-ms': { type: 'string', default: '1500' },
|
||||
},
|
||||
});
|
||||
|
||||
const run = new Run(createLLM(loadProvider()), Number(values.level));
|
||||
const t0 = Date.now();
|
||||
const stamp = () => `${((Date.now() - t0) / 1000).toFixed(1).padStart(5)}s`;
|
||||
const log = (text: string) => console.log(`${stamp()} ${text}`);
|
||||
const short = (text: string, n = 110) => text.replace(/\s+/g, ' ').slice(0, n);
|
||||
|
||||
let busy = false;
|
||||
let turnIdleSince = 0;
|
||||
let streamed = '';
|
||||
|
||||
const [answerWhen, ...answerRest] = (values.answer ?? '').split(':');
|
||||
const answerText = answerRest.join(':');
|
||||
|
||||
run.subscribe((event: TraceEvent) => {
|
||||
switch (event.type) {
|
||||
case 'model.start':
|
||||
busy = true;
|
||||
streamed = '';
|
||||
log(`▶ 模型请求 #${event.n}（历史 ${event.messageCount} 条）`);
|
||||
break;
|
||||
case 'model.delta':
|
||||
if (event.content) streamed += event.content;
|
||||
break;
|
||||
case 'model.end': {
|
||||
const calls = event.message.tool_calls?.map((c) => `${c.function.name}(${short(c.function.arguments, 70)})`) ?? [];
|
||||
log(`◀ 模型返回 #${event.n} ttft=${event.ttftMs}ms ${short(streamed)}${calls.length ? `\n 工具调用：${calls.join(' | ')}` : ''}`);
|
||||
if (!calls.length) {
|
||||
busy = false;
|
||||
turnIdleSince = Date.now();
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 'model.error':
|
||||
log(`✖ 模型错误 #${event.n} ${event.error}`);
|
||||
break;
|
||||
case 'tool.start':
|
||||
log(` ├ 工具开始 ${event.name} ${event.callId.slice(-6)}`);
|
||||
break;
|
||||
case 'tool.end':
|
||||
log(` └ 工具返回 ${event.callId.slice(-6)} ${short(event.result, 90)}`);
|
||||
break;
|
||||
case 'job.start':
|
||||
log(` ◆ 后台任务 ${event.jobId} 启动：${event.command}`);
|
||||
break;
|
||||
case 'job.end':
|
||||
log(` ◆ 后台任务 ${event.jobId} 退出 exit=${event.exitCode} ${(event.durationMs / 1000).toFixed(1)}s`);
|
||||
break;
|
||||
case 'question.ask':
|
||||
log(` ? 提问 ${event.questionId}（${event.mode}）${event.question} [${event.options.join(' / ')}]`);
|
||||
scheduleAnswer(event.questionId);
|
||||
break;
|
||||
case 'question.answer':
|
||||
log(` ! 用户回答 ${event.questionId}：${event.answer}`);
|
||||
break;
|
||||
case 'inbox.push':
|
||||
log(` ✉ 收件箱 +${event.item.kind}${event.item.ref ? `(${event.item.ref})` : ''}`);
|
||||
break;
|
||||
case 'inbox.deliver':
|
||||
log(` ✉ 收件箱 → 历史：${event.ids.join(', ')}`);
|
||||
break;
|
||||
case 'run.stop':
|
||||
if (event.reason === 'error') log(`✖ 运行出错：${event.error}`);
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
function scheduleAnswer(questionId: string) {
|
||||
if (!answerWhen) return;
|
||||
const fire = () => run.agent.answerQuestion(questionId, answerText);
|
||||
if (answerWhen === 'idle') {
|
||||
const timer = setInterval(() => {
|
||||
if (!busy && turnIdleSince && Date.now() - turnIdleSince > 1500) {
|
||||
clearInterval(timer);
|
||||
fire();
|
||||
}
|
||||
}, 200);
|
||||
} else {
|
||||
setTimeout(fire, Number(answerWhen));
|
||||
}
|
||||
}
|
||||
|
||||
if (values.steer) {
|
||||
const [when, ...rest] = values.steer.split(':');
|
||||
const text = rest.join(':');
|
||||
const fire = () => {
|
||||
log(` ✉ 用户插话：${text}`);
|
||||
run.agent.sendUserMessage(text);
|
||||
};
|
||||
if (when.startsWith('test+')) {
|
||||
const delay = Number(when.slice(5));
|
||||
let armed = true;
|
||||
run.subscribe((event) => {
|
||||
const isTest = (event.type === 'tool.start' && /npm (run )?test/.test(String((event.args as any)?.command ?? ''))) ||
|
||||
(event.type === 'job.start' && /npm (run )?test/.test(event.command));
|
||||
if (armed && isTest) {
|
||||
armed = false;
|
||||
setTimeout(fire, delay);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
setTimeout(fire, Number(when));
|
||||
}
|
||||
}
|
||||
|
||||
log(`运行 ${run.runId}，工作区 ${run.workspace}`);
|
||||
run.start();
|
||||
|
||||
const idleMs = Number(values['idle-ms']);
|
||||
await new Promise<void>((resolve) => {
|
||||
const timer = setInterval(() => {
|
||||
const tools = run.agent.tools;
|
||||
const settled =
|
||||
!busy && turnIdleSince && Date.now() - turnIdleSince > idleMs &&
|
||||
tools.runningJobs.size === 0 && tools.pendingQuestions.size === 0;
|
||||
const failed = run.events.some((e) => e.type === 'run.stop');
|
||||
if (settled || failed) {
|
||||
clearInterval(timer);
|
||||
resolve();
|
||||
}
|
||||
}, 300);
|
||||
});
|
||||
|
||||
run.stop();
|
||||
const lastDiff = [...run.events].reverse().find((e) => e.type === 'workspace.diff');
|
||||
console.log('\n===== 最终 diff =====\n' + (lastDiff && 'diff' in lastDiff ? lastDiff.diff : '(无)'));
|
||||
const dir = join(PROJECT_ROOT, 'recordings');
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const file = join(dir, `${values.out ?? run.runId}.json`);
|
||||
writeFileSync(file, JSON.stringify(run.recording(), null, 1));
|
||||
console.log(`录制已保存：${file}`);
|
||||
process.exit(0);
|
||||
26
async-agent-lab/scripts/show.ts
Normal file
26
async-agent-lab/scripts/show.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
// 把一份录制打印成紧凑时间线：node scripts/show.ts recordings/level2-background.json
|
||||
import { readFileSync } from 'node:fs';
|
||||
import type { RunRecording } from '../shared/types.ts';
|
||||
|
||||
const rec: RunRecording = JSON.parse(readFileSync(process.argv[2], 'utf8'));
|
||||
const s = (x: unknown, n = 100) => String(typeof x === 'string' ? x : JSON.stringify(x)).replace(/\s+/g, ' ').slice(0, n);
|
||||
const skip = new Set(['model.delta', 'tool.output', 'job.output', 'history.append', 'workspace.diff', 'run.start']);
|
||||
for (const e of rec.events) {
|
||||
if (skip.has(e.type)) continue;
|
||||
let d = '';
|
||||
switch (e.type) {
|
||||
case 'model.start': d = `#${e.n} 历史 ${e.messageCount} 条`; break;
|
||||
case 'model.end': d = `#${e.n} ${s(e.message.content, 80)} ${(e.message.tool_calls ?? []).map((c) => `${c.function.name}${s(c.function.arguments, 50)}`).join(' | ')}`; break;
|
||||
case 'tool.start': d = `${e.name} ${s(e.args, 70)}`; break;
|
||||
case 'tool.end': d = s(e.result, 80); break;
|
||||
case 'job.start': d = `${e.jobId} ${e.command}`; break;
|
||||
case 'job.end': d = `${e.jobId} exit=${e.exitCode} ${e.durationMs}ms`; break;
|
||||
case 'inbox.push': d = `${e.item.id} ${e.item.kind} ${e.item.ref ?? ''}`; break;
|
||||
case 'inbox.deliver': d = e.ids.join(','); break;
|
||||
case 'question.ask': d = `${e.questionId} ${e.mode} ${e.question} [${e.options.join('/')}]`; break;
|
||||
case 'question.answer': d = `${e.questionId} ${e.answer}`; break;
|
||||
case 'model.error': d = e.error; break;
|
||||
case 'run.stop': d = e.reason; break;
|
||||
}
|
||||
console.log(`${(e.t / 1000).toFixed(1).padStart(6)} ${e.type.padEnd(15)} ${d}`);
|
||||
}
|
||||
65
async-agent-lab/scripts/stats.ts
Normal file
65
async-agent-lab/scripts/stats.ts
Normal file
@@ -0,0 +1,65 @@
|
||||
// 从录制里算出分享文档要用的关键数字：node scripts/stats.ts
|
||||
import { readdirSync, readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { PROJECT_ROOT } from '../agent/config.ts';
|
||||
import type { RunRecording, TraceEvent } from '../shared/types.ts';
|
||||
|
||||
const dir = join(PROJECT_ROOT, 'recordings');
|
||||
const sec = (ms?: number) => (ms === undefined ? '-' : `${(ms / 1000).toFixed(1)}s`);
|
||||
|
||||
for (const file of readdirSync(dir).filter((f) => f.endsWith('.json')).sort()) {
|
||||
const rec: RunRecording = JSON.parse(readFileSync(join(dir, file), 'utf8'));
|
||||
const ev = rec.events;
|
||||
const of = <T extends TraceEvent['type']>(type: T) => ev.filter((e) => e.type === type) as Extract<TraceEvent, { type: T }>[];
|
||||
const reqs = of('model.start');
|
||||
const ends = of('model.end');
|
||||
const tools = of('tool.start').map((s) => ({ s, e: of('tool.end').find((e) => e.callId === s.callId) }));
|
||||
const jobs = of('job.start').map((s) => ({ s, e: of('job.end').find((e) => e.jobId === s.jobId) }));
|
||||
const asks = of('question.ask');
|
||||
const answers = of('question.answer');
|
||||
const pushes = of('inbox.push');
|
||||
const delivers = of('inbox.deliver');
|
||||
const deliverAt = (id: string) => delivers.find((d) => d.ids.includes(id))?.t;
|
||||
const nextReq = (t?: number) => (t === undefined ? undefined : reqs.find((r) => r.t >= t));
|
||||
const tokens = ends.reduce((a, e) => a + (e.usage?.total_tokens ?? 0), 0);
|
||||
const ttft = ends.map((e) => e.ttftMs ?? 0);
|
||||
const reqDur = ends.map((e) => e.t - reqs.find((r) => r.n === e.n)!.t);
|
||||
|
||||
console.log(`\n## ${file}（第 ${rec.level} 级）总时长 ${sec(ev.at(-1)!.t)}，模型请求 ${reqs.length} 次，tokens ${tokens}`);
|
||||
console.log(` 模型请求耗时 中位 ${sec(median(reqDur))}，首包中位 ${median(ttft)}ms`);
|
||||
for (const { s, e } of tools) {
|
||||
const cmd = (s.args as any)?.command;
|
||||
if (e && e.t - s.t > 1000) console.log(` 工具 ${s.name} 阻塞 ${sec(s.t)}→${sec(e.t)}（${sec(e.t - s.t)}）${cmd ?? ''}`);
|
||||
}
|
||||
for (const { s, e } of jobs) {
|
||||
const during = reqs.filter((r) => r.t >= s.t && r.t < (e?.t ?? Infinity)).map((r) => `#${r.n}`);
|
||||
const notice = pushes.find((p) => p.item.kind === 'job_done' && p.item.ref === s.jobId);
|
||||
const seen = nextReq(deliverAt(notice?.item.id ?? ''));
|
||||
console.log(` 后台 ${s.jobId} ${sec(s.t)}→${sec(e?.t)}（${sec(e && e.t - s.t)}），工具当场返回；运行期间请求 ${during.join(' ') || '无'}；完成通知被 #${seen?.n} 在 ${sec(seen?.t)} 看到`);
|
||||
}
|
||||
for (const a of asks) {
|
||||
const tEnd = of('tool.end').find((e) => e.callId === a.callId);
|
||||
const ans = answers.find((x) => x.questionId === a.questionId);
|
||||
const during = reqs.filter((r) => r.t >= a.t && r.t < (ans?.t ?? Infinity)).map((r) => `#${r.n}`);
|
||||
const reply = pushes.find((p) => p.item.kind === 'question_reply' && p.item.ref === a.questionId);
|
||||
const seen = a.mode === 'blocking' ? nextReq(tEnd?.t) : nextReq(deliverAt(reply?.item.id ?? ''));
|
||||
const firstEdit = of('tool.start').find((t) => t.name === 'edit_file');
|
||||
console.log(` 提问 ${a.questionId}（${a.mode}）${sec(a.t)} 发出，工具 ${sec(tEnd?.t)} 返回；${sec(ans?.t)} 回答「${ans?.answer}」；挂着期间请求 ${during.join(' ') || '无'}；答案被 #${seen?.n} 在 ${sec(seen?.t)} 看到；第一次改文件 ${sec(firstEdit?.t)}`);
|
||||
// 回答前最后一次模型请求说了什么
|
||||
const before = ends.filter((x) => x.t < (ans?.t ?? Infinity)).at(-1);
|
||||
if (before && !before.message.tool_calls?.length) console.log(` 回答前停在：${before.message.content.replace(/\s+/g, ' ').slice(0, 160)}`);
|
||||
}
|
||||
for (const p of pushes.filter((p) => p.item.kind === 'user' && p.item.id !== 'in_1')) {
|
||||
const d = deliverAt(p.item.id);
|
||||
const seen = nextReq(d);
|
||||
console.log(` 插话 ${sec(p.t)} 进收件箱，${sec(d)} 加进历史（排队 ${sec(d === undefined ? undefined : d - p.t)}），#${seen?.n} 看到`);
|
||||
}
|
||||
const diff = of('workspace.diff').at(-1)?.diff ?? '';
|
||||
const keeps = /^\+.*status: order\.state/m.test(diff) || !/^-\s+status: order\.state/m.test(diff);
|
||||
console.log(` 结果：返回体${keeps ? '保留' : '移除'}了旧字段 status；改动文件 ${(diff.match(/^diff --git/gm) ?? []).length} 个`);
|
||||
}
|
||||
|
||||
function median(xs: number[]) {
|
||||
const s = [...xs].sort((a, b) => a - b);
|
||||
return s.length ? s[Math.floor(s.length / 2)] : 0;
|
||||
}
|
||||
Reference in New Issue
Block a user