[rig-decomposition-bench] Daily rig decomposition benchmark — 2026-10-14 — no winner (task picker failed)

import { agent, workflow, s, configureAgent, copilotEngine, repair } from "rig";

configureAgent(copilotEngine());

const DEADLINE = Date.now() + 25 * 60_000;
const remaining = () => DEADLINE - Date.now();

const cp: any = (globalThis as any).process.getBuiltinModule("node:child_process");

type RunResult = { status: "passed" | "failed" | "skipped"; output: string; stdout: string };

function runCli(source: string, flag: "--typecheck" | "--server", wantMs: number): Promise<RunResult> {
  const budget = Math.min(wantMs, remaining());
  if (budget < 5_000) {
    return Promise.resolve({ status: "skipped", output: "time budget exhausted", stdout: "" });
  }
  return new Promise((resolve) => {
    const child = cp.spawn("node", ["rig.ts", flag], { cwd: (globalThis as any).process.cwd(), stdio: ["pipe", "pipe", "pipe"] });
    let out = "";
    let err = "";
    let timedOut = false;
    const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, budget);
    child.stdout.on("data", (d: any) => { out += String(d); });
    child.stderr.on("data", (d: any) => { err += String(d); });
    child.on("error", (e: any) => { clearTimeout(timer); resolve({ status: "failed", output: String(e), stdout: out }); });
    child.on("close", (code: number | null) => {
      clearTimeout(timer);
      const combined = (err + (code === 0 ? "" : out)).trim();
      if (timedOut) resolve({ status: "failed", output: `timed out after ${budget}ms\n${combined}`, stdout: out });
      else if (code === 0) resolve({ status: "passed", output: combined, stdout: out });
      else resolve({ status: "failed", output: `exit code ${code}\n${combined}`, stdout: out });
    });
    child.stdin.end(source);
  });
}

function parseSolution(stdout: string): string | null {
  const t = stdout.trim();
  const tryParse = (x: string) => {
    try {
      const v = JSON.parse(x);
      if (v && typeof v === "object" && typeof v.solution === "string") return v.solution as string;
    } catch { /* ignore */ }
    return null;
  };
  const direct = tryParse(t);
  if (direct !== null) return direct;
  const i = t.indexOf("{");
  const j = t.lastIndexOf("}");
  if (i >= 0 && j > i) return tryParse(t.slice(i, j + 1));
  return null;
}

// Agent role: picks a concrete, gradable multi-faceted task
const pickTask = agent({
  model: "small",
  maxTurns: 2,
  addons: [repair()],
  instructions:
    "Pick ONE concrete, complicated task a person or team might face in a single day that would naturally benefit from being split across sub-agents running different models: multi-source research and synthesis, a multi-file coding task with design/implementation/review phases, a structured report combining several independent analyses, or a multi-step data transformation pipeline. It must be self-contained (solvable from its description alone, no external files or live web access) and concrete enough to grade. Vary the domain from run to run (use the current time as a random seed). Return a title, domain, description, and 3-6 concrete, checkable success criteria.",
  output: s.object({
    title: s.string,
    domain: s.string,
    description: s.string,
    criteria: s.array(s.string),
  }),
});

// Agent role: solves the whole task in one turn
const solveSingle = agent({
  model: "medium",
  maxTurns: 1,
  instructions:
    "Solve the entire task yourself in a single response, addressing every success criterion. No delegation, no tools. Return the complete solution verbatim (Markdown code fences are fine).",
  output: s.string,
});

const SKELETON = `import { agent, workflow, s } from "rig";

// Agent role: <role>
const analyze = agent({ model: "small", instructions: "...", output: s.string });

// Workflow role: combines the agents' outputs into the final solution
export default workflow({
  meta: { name: "decomposed-solution", description: "..." },
  body: async ({ call }) => {
    const analysis = await call(analyze, "...");
    return { solution: analysis ?? "" };
  },
});`;

const CONTRACT = `Write ONE self-contained TypeScript program using the "rig" library. Rules:
- Import only from "rig". Take no required input on the root. Export the root via \`export default\` and never invoke it.
- Split the task across AT LEAST TWO agents. Put a \`// Agent role: ...\` comment above each agent() and a \`// Workflow role: ...\` comment above the workflow(). Each agent uses model "small" for simple sub-steps and "medium" or "large" for harder ones. The final answer must combine the agents' outputs rather than come from one agent solving everything.
- A workflow() spec accepts ONLY \`meta\`, an optional \`input\`, and \`body\`. It has no \`output\` or \`execute\` field; the final { solution } object is simply what \`body\` returns. (An agent() root instead declares \`output: s.object({ solution: s.string })\`.) Inside body, \`call(agent, input)\` returns the output or null on failure; \`parallel([...thunks])\` runs thunks concurrently.
- Use \`output: s.string\` for every agent whose result is source text or free-form solution. Do not make any agent re-emit a large solution inside an object wrapper.
- Do not use console.log.
Required shape (abridged to one agent; you need at least two):

\`\`\`ts
${SKELETON}
\`\`\`

Return ONLY the complete program source as plain text, with no commentary.`;

// Agent role: writes a decomposed rig program
const writeProgram = agent({
  model: "medium",
  maxTurns: 1,
  instructions: CONTRACT,
  output: s.string,
});

// Agent role: fixes or regenerates the decomposed rig program
const fixProgram = agent({
  model: "medium",
  maxTurns: 1,
  instructions: CONTRACT + "\nYou may be given a failing program with the exact error; fix precisely what the error names and preserve what already worked. If told there is no source to fix, generate the complete program from the task and skeleton.",
  output: s.string,
});

// Agent role: grades both solutions on content
const grade = agent({
  model: "large",
  maxTurns: 1,
  instructions:
    "Score each solution 0-10 on how completely and correctly it satisfies the success criteria. Judge only correctness and completeness of content — not length, and not which approach produced it. Pick winner single-call, decomposed, or tie, and explain the comparison.",
  output: s.object({
    singleScore: s.number,
    decomposedScore: s.number,
    winner: s.enum("single-call", "decomposed", "tie"),
    rationale: s.string,
  }),
});

function stripFence(src: string): string {
  const m = src.match(/```(?:ts|typescript|rig)?\s*\n([\s\S]*?)```/);
  return (m ? m[1] : src).trim();
}

type Attempt = {
  attempt: number;
  generation: string;
  typecheck: string;
  execute: string;
  error: string;
};

// Workflow role: picks a task, solves it two ways, grades both and returns the report
export default workflow({
  meta: { name: "rig-decomposition-bench", description: "Single-call vs decomposed benchmark" },
  body: async ({ call }) => {
    const task = await call(pickTask, "Pick today's task.");
    if (!task) {
      return { status: "failed", error: "task picker returned null", task: null };
    }
    const taskText = `# ${task.title}\nDomain: ${task.domain}\n\n${task.description}\n\nSuccess criteria:\n${task.criteria.map((c: string) => `- ${c}`).join("\n")}`;

    const t0 = Date.now();
    const single = await call(solveSingle, taskText, { timeout: Math.max(10_000, Math.min(8 * 60_000, remaining())) });
    const singleMs = Date.now() - t0;

    const t1 = Date.now();
    const attempts: Attempt[] = [];
    let source = "";
    let decomposedSolution: string | null = null;
    let status = "failed";
    let lastSource = "";
    let lastError = "";
    for (let n = 1; n <= 2; n++) {
      if (remaining() < 4 * 60_000 && n > 1) break;
      if (remaining() < 60_000) break;
      const writerBudget = Math.max(10_000, Math.min(5 * 60_000, remaining()));
      let raw: string | null;
      if (n === 1) {
        raw = await call(writeProgram, `Task:\n${taskText}`, { timeout: writerBudget });
      } else if (lastSource.trim()) {
        raw = await call(fixProgram, `Task:\n${taskText}\n\nThe following program failed.\n\nProgram:\n${lastSource}\n\nExact error:\n${lastError}\n\nFix precisely what the error names, preserving what already worked.`, { timeout: writerBudget });
      } else {
        raw = await call(fixProgram, `Task:\n${taskText}\n\nThere is no source to fix: the previous attempt produced nothing. Generate the complete program from the task and the required skeleton.`, { timeout: writerBudget });
      }
      const rec: Attempt = { attempt: n, generation: "", typecheck: "skipped", execute: "skipped", error: "" };
      attempts.push(rec);
      if (raw === null || raw === undefined || !raw.trim()) {
        rec.generation = "empty";
        lastSource = "";
        lastError = "writer returned empty result";
        rec.error = lastError;
        continue;
      }
      rec.generation = "ok";
      const src = stripFence(raw);
      lastSource = src;
      source = src;
      const tc = await runCli(src, "--typecheck", 2 * 60_000);
      rec.typecheck = tc.status;
      if (tc.status !== "passed") {
        lastError = tc.output;
        rec.error = tc.output;
        continue;
      }
      const ex = await runCli(src, "--server", 5 * 60_000);
      rec.execute = ex.status;
      if (ex.status !== "passed") {
        lastError = ex.output;
        rec.error = ex.output;
        continue;
      }
      const sol = parseSolution(ex.stdout);
      if (sol === null || !sol.trim()) {
        rec.execute = "failed";
        lastError = "could not parse non-empty { solution } from stdout:\n" + ex.stdout.slice(0, 4000);
        rec.error = lastError;
        continue;
      }
      decomposedSolution = sol;
      status = "passed";
      break;
    }
    const decomposedMs = Date.now() - t1;

    const singleText = single && single.trim() ? single : "(single-call solver returned nothing)";
    const decText = decomposedSolution ?? "(decomposition produced no solution)";
    const grading = await call(
      grade,
      `${taskText}\n\n=== SOLUTION A (single-call) ===\n${singleText}\n\n=== SOLUTION B (decomposed) ===\n${decText}`,
      { timeout: Math.max(10_000, Math.min(5 * 60_000, remaining() + 60_000)) },
    );

    return {
      status,
      task,
      singleMs,
      decomposedMs,
      singleSolution: singleText,
      decomposedSolution: decText,
      attempts,
      decomposedSource: source,
      grading,
    };
  },
});