[rig-decomposition-bench] Daily rig decomposition benchmark — 2026-10-29 — no result (task picker failed)

import { agent, workflow, s, repair, configureAgent, copilotEngine } from "rig";

configureAgent(copilotEngine());

const DEADLINE = Date.now() + 25 * 60_000;
const remaining = () => DEADLINE - Date.now();

type RunResult = { code: number | null; stdout: string; stderr: string; timedOut: boolean };

async function runRig(source: string, flag: "--typecheck" | "--server", timeoutMs: number): Promise<RunResult> {
  const { spawn } = await import("node:child_process");
  return new Promise<RunResult>((resolve) => {
    const child = spawn("node", ["rig.ts", flag], { cwd: process.cwd(), stdio: ["pipe", "pipe", "pipe"] });
    let stdout = "";
    let stderr = "";
    let timedOut = false;
    let done = false;
    const finish = (code: number | null) => {
      if (done) return;
      done = true;
      clearTimeout(timer);
      resolve({ code, stdout, stderr, timedOut });
    };
    const timer = setTimeout(() => {
      timedOut = true;
      child.kill("SIGKILL");
      setTimeout(() => finish(null), 2000);
    }, Math.max(1000, timeoutMs));
    child.stdout.on("data", (d: Buffer) => { stdout += d.toString(); });
    child.stderr.on("data", (d: Buffer) => { stderr += d.toString(); });
    child.on("error", (e: Error) => { stderr += String(e); finish(-1); });
    child.on("close", (code: number | null) => finish(code));
    child.stdin.on("error", () => {});
    child.stdin.end(source);
  });
}

// Agent role: picks a benchmark task
const picker = agent({
  name: "picker",
  model: "small",
  maxTurns: 2,
  addons: [repair()],
  instructions:
    "Pick one concrete, complicated task a person or team might face in a single day that would naturally benefit from being split across sub-agents running different models: multi-source research and synthesis, a multi-file coding task with design/implementation/review phases, a structured report combining several independent analyses, or a multi-step data transformation pipeline. It must be self-contained (solvable from its description alone, no external files or live web access) and concrete enough to grade. Vary the domain (pick a random, unusual one). Return a title, a domain, a description, and 3-6 concrete checkable success criteria.",
  output: s.object({
    title: s.string,
    domain: s.string,
    description: s.string,
    criteria: s.array(s.string),
  }),
});

// Agent role: solves the whole task alone in one turn
const solver = agent({
  name: "solver",
  model: "medium",
  maxTurns: 1,
  instructions:
    "Solve the entire task yourself in a single response, addressing every success criterion. No delegation, no tools. Return the complete solution verbatim as text (Markdown code fences allowed).",
  output: s.string,
});

const SKELETON = `import { agent, workflow, s } from "rig";

// Agent role: <role>
const analyze = agent({ model: "small", instructions: "...", output: s.string });

// Workflow role: combines the agents' outputs into the final solution
export default workflow({
  meta: { name: "decomposed-solution", description: "..." },
  body: async ({ call }) => {
    const analysis = await call(analyze, "...");
    return { solution: analysis ?? "" };
  },
});`;

const CONTRACT = `Write a complete, self-contained rig TypeScript program (no Markdown fences, source only) that solves the task by splitting it across AT LEAST TWO agents, coordinated so the final answer combines their outputs rather than one agent solving everything.
Rules:
- Import only from "rig". Use named imports such as agent, workflow, s.
- Each agent() has a "// Agent role: ..." comment above it; choose model "small" for simple sub-steps and "medium" or "large" for harder ones.
- Root takes no required input and is exported with "export default" and NOT invoked.
- A workflow() spec accepts only meta, an optional input, and body. It has NO output or execute field; the final { solution } object is simply what body returns. Inside body use the destructured helper call(agent, inputString) which returns the agent output or null on failure. An agent() root would instead declare output: s.object({ solution: s.string }).
- Use s.string for every agent whose result is source text or free-form solution. Do not make any agent re-emit a large solution inside a wrapper object.
- Do not use console.log. Put the task text directly in the agents' instructions or inputs.
Required shape (abridged to one agent; yours needs at least two):
${SKELETON}`;

// Agent role: writes the decomposed rig program
const writer = agent({
  name: "writer",
  model: "medium",
  maxTurns: 1,
  instructions: CONTRACT,
  output: s.string,
});

// Agent role: fixes or regenerates the decomposed program
const fixer = agent({
  name: "fixer",
  model: "medium",
  maxTurns: 1,
  instructions: CONTRACT + "\nYou may be asked to fix an existing program given an error, or to generate it from scratch. Fix precisely what the error names and preserve what already worked. Return only the complete program source.",
  output: s.string,
});

// Agent role: grades both solutions
const grader = agent({
  name: "grader",
  model: "large",
  maxTurns: 1,
  instructions:
    "Score each solution 0-10 on how completely and correctly it satisfies the success criteria. Judge only correctness and completeness of content, not length and not which approach produced it. Pick winner 'single-call', 'decomposed', or 'tie' and explain the comparison.",
  output: s.object({
    singleScore: s.number,
    decomposedScore: s.number,
    winner: s.enum("single-call", "decomposed", "tie"),
    rationale: s.string,
  }),
});

function stripFences(src: string): string {
  const m = src.match(/```(?:ts|typescript|rig)?\n([\s\S]*?)```/);
  return (m ? m[1] : src).trim();
}

type Attempt = {
  attempt: number;
  generation: string;
  typecheck: string;
  execute: string;
  error: string;
  note: string;
};

export default workflow({
  meta: { name: "rig-decomposition-bench", description: "Single-call vs decomposed benchmark" },
  body: async ({ call }) => {
    const task = await call(picker, "Pick today's benchmark task.");
    if (!task) return { status: "fail: task picker returned null", task: null };
    const taskText = `Title: ${task.title}\nDomain: ${task.domain}\nDescription: ${task.description}\nSuccess criteria:\n${task.criteria.map((c: string, i: number) => `${i + 1}. ${c}`).join("\n")}`;

    const t0 = Date.now();
    const single = await call(solver, taskText, { timeout: Math.max(1000, Math.min(8 * 60_000, remaining())) });
    const singleMs = Date.now() - t0;

    const t1 = Date.now();
    const attempts: Attempt[] = [];
    let source = "";
    let decomposedSolution = "";
    let status = "fail";
    let prevSource = "";
    let prevError = "";
    for (let n = 1; n <= 2; n++) {
      if (remaining() < 3 * 60_000) break;
      const a: Attempt = { attempt: n, generation: "", typecheck: "skipped", execute: "skipped", error: "", note: "" };
      attempts.push(a);
      let out: string | null;
      const gen = Math.max(1000, Math.min(6 * 60_000, remaining()));
      if (n === 1) {
        out = await call(writer, `Task:\n${taskText}`, { timeout: gen });
      } else if (prevSource.trim()) {
        out = await call(fixer, `Task:\n${taskText}\n\nCurrent program:\n${prevSource}\n\nExact error captured:\n${prevError}\n\nFix precisely what the error names.`, { timeout: gen });
      } else {
        out = await call(fixer, `Task:\n${taskText}\n\nThere is no source to fix: the previous attempt produced nothing. Generate the complete program from the task and skeleton.`, { timeout: gen });
      }
      const src = out && out.trim() ? stripFences(out) : "";
      if (!src) {
        a.generation = "empty/null";
        a.error = "writer returned null or whitespace-only source";
        prevSource = "";
        prevError = a.error;
        continue;
      }
      a.generation = "ok";
      source = src;
      prevSource = src;
      if (remaining() < 30_000) { a.error = "time budget exhausted"; break; }
      const tc = await runRig(src, "--typecheck", Math.min(2 * 60_000, remaining()));
      if (tc.code !== 0) {
        a.typecheck = tc.timedOut ? "timeout" : "fail";
        a.error = (tc.stdout + "\n" + tc.stderr).trim();
        prevError = a.error;
        continue;
      }
      a.typecheck = "pass";
      if (remaining() < 30_000) { a.error = "time budget exhausted"; break; }
      const ex = await runRig(src, "--server", Math.min(5 * 60_000, remaining()));
      if (ex.code !== 0) {
        a.execute = ex.timedOut ? "timeout" : "fail";
        a.error = (ex.stdout + "\n" + ex.stderr).trim();
        prevError = a.error;
        continue;
      }
      try {
        const parsed = JSON.parse(ex.stdout.trim());
        decomposedSolution = typeof parsed?.solution === "string" ? parsed.solution : JSON.stringify(parsed);
        a.execute = "pass";
        status = "pass";
        break;
      } catch {
        a.execute = "fail";
        a.error = "could not parse JSON stdout:\n" + ex.stdout + "\n" + ex.stderr;
        prevError = a.error;
      }
    }
    for (let i = 0; i < attempts.length - 1; i++) {
      attempts[i].note = attempts[i].generation === "ok" ? "next attempt asked the fixer to fix the captured error" : "next attempt regenerated from scratch";
    }
    const decomposedMs = Date.now() - t1;

    const grade = await call(
      grader,
      `${taskText}\n\n=== SOLUTION A (single-call) ===\n${single ?? "(no solution produced)"}\n\n=== SOLUTION B (decomposed) ===\n${decomposedSolution || "(decomposition produced no solution)"}`,
      { timeout: Math.max(1000, Math.min(6 * 60_000, remaining() + 60_000)) },
    );

    return { task, singleMs, decomposedMs, single: single ?? "", decomposedSolution, attempts, status, source, grade };
  },
});