[rig-decomposition-bench] Daily rig decomposition benchmark — 2026-10-24 — no winner (picker failed)

import { agent, workflow, s, configureAgent, copilotEngine, repair } from "rig";

configureAgent(copilotEngine());

const DEADLINE = Date.now() + 25 * 60_000;
const left = () => DEADLINE - Date.now();

const SKELETON = `import { agent, workflow, s } from "rig";

// Agent role: <role>
const analyze = agent({ model: "small", instructions: "...", output: s.string });

// Workflow role: combines the agents' outputs into the final solution
export default workflow({
  meta: { name: "decomposed-solution", description: "..." },
  body: async ({ call }) => {
    const analysis = await call(analyze, "...");
    return { solution: analysis ?? "" };
  },
});`;

const RULES = `Rules for the rig program (you have no access to the library docs, follow these exactly):
- Import only from "rig". Export the root via \`export default\` and never invoke it. The root takes no required input.
- A workflow() spec accepts only meta, an optional input, and body. It has NO output or execute field; the final { solution } object is simply what body returns.
- An agent() root instead declares output: s.object({ solution: s.string }).
- agent() fields: name, instructions, input, output, model ("small" | "medium" | "large"), maxTurns, addons, agents, tools.
- Inside body({ call, parallel, pipeline }): call(agent, input) returns the output or null on failure; parallel([thunks]) runs thunks concurrently.
- Use at least two agents, each preceded by a "// Agent role: ..." comment; use "small" for simple sub-steps, "medium" or "large" for harder ones. Put a "// Workflow role: ..." comment above the workflow.
- The final answer must combine the agents' outputs, not come from one agent solving everything.
- Use s.string for every agent whose result is source text or free-form solution; do not make agents re-emit a large solution inside an object wrapper.
- Do not use console.log. Return ONLY the complete TypeScript source, optionally in one \`\`\`ts fence.
Required shape (abridged to one agent; you need at least two):
${SKELETON}`;

const picker = agent({
  name: "picker",
  model: "small",
  maxTurns: 2,
  addons: [repair()],
  instructions:
    "Pick ONE concrete, complicated task a person or team might face in a single day that would naturally benefit from being split across sub-agents running different models: multi-source research and synthesis, a multi-file coding task with design/implementation/review phases, a structured report combining independent analyses, or a multi-step data transformation pipeline. It must be self-contained (solvable from its description alone, no external files or live web access) and concrete enough to grade. Vary the domain; pick randomly. Return title, domain, description, and 3-6 concrete checkable success criteria.",
  output: s.object({
    title: s.string,
    domain: s.string,
    description: s.string,
    criteria: s.array(s.string),
  }),
});

// Agent role: solves the whole task alone in one turn
const solver = agent({
  name: "solver",
  model: "medium",
  maxTurns: 1,
  instructions:
    "Solve the entire task yourself in one response, addressing every success criterion explicitly. No delegation, no tools. Return the complete solution verbatim.",
  output: s.string,
});

// Agent role: writes (and fixes) the decomposed rig program
const writer = agent({
  name: "writer",
  model: "medium",
  maxTurns: 1,
  instructions:
    "You write self-contained rig TypeScript programs that solve a task by splitting it across multiple agents.\n" + RULES,
  output: s.string,
});

// Agent role: grades both solutions
const grader = agent({
  name: "grader",
  model: "large",
  maxTurns: 1,
  instructions:
    "Score each solution 0-10 on how completely and correctly it satisfies the success criteria. Judge only correctness and completeness of content, not length and not which approach produced it. Pick winner single-call, decomposed, or tie and explain the comparison.",
  output: s.object({
    singleScore: s.number,
    decomposedScore: s.number,
    winner: s.enum("single-call", "decomposed", "tie"),
    rationale: s.string,
  }),
});

type Attempt = {
  attempt: number;
  generation: string;
  typecheck: string;
  execute: string;
  error: string;
};

function stripFence(src: string): string {
  const m = src.match(/```(?:ts|typescript|rig)?\s*\n([\s\S]*?)```/);
  return (m ? m[1] : src).trim();
}

async function runCli(args: string[], input: string, timeoutMs: number) {
  const modName: string = "node:child_process";
  const { spawn } = (await import(modName)) as any;
  const proc = (globalThis as any).process;
  return new Promise<{ code: number | null; stdout: string; stderr: string; timedOut: boolean }>((resolve) => {
    const child = spawn("node", ["rig.ts", ...args], { cwd: proc.cwd() });
    let stdout = "";
    let stderr = "";
    let timedOut = false;
    const t = setTimeout(() => {
      timedOut = true;
      child.kill("SIGKILL");
    }, Math.max(1000, timeoutMs));
    child.stdout.on("data", (d: any) => (stdout += d.toString()));
    child.stderr.on("data", (d: any) => (stderr += d.toString()));
    child.on("error", (e: Error) => {
      clearTimeout(t);
      resolve({ code: -1, stdout, stderr: stderr + String(e), timedOut });
    });
    child.on("close", (code: number | null) => {
      clearTimeout(t);
      resolve({ code, stdout, stderr, timedOut });
    });
    child.stdin.on("error", () => {});
    child.stdin.end(input);
  });
}

// Workflow role: pick a task, solve it two ways, grade, and return the report
export default workflow({
  meta: { name: "decomposition-bench", description: "Single-call vs decomposed rig solution benchmark" },
  body: async ({ call }) => {
    const task = await call(picker, "Pick the task now.", { label: "pick" });
    if (!task) return { error: "task picker returned null", status: "fail" };
    const taskText = `Title: ${task.title}\nDomain: ${task.domain}\nDescription: ${task.description}\nSuccess criteria:\n${task.criteria.map((c: string, i: number) => `${i + 1}. ${c}`).join("\n")}`;

    const t0 = Date.now();
    const singleSolution = (await call(solver, taskText, { label: "single", timeout: Math.min(8 * 60_000, Math.max(30_000, left())) })) ?? "";
    const singleMs = Date.now() - t0;

    const d0 = Date.now();
    const attempts: Attempt[] = [];
    let source = "";
    let decomposedSolution = "";
    let status = "fail";
    let prevSource = "";
    let prevError = "";
    for (let n = 1; n <= 2; n++) {
      if (left() < 4 * 60_000) break;
      let prompt: string;
      if (n === 1) {
        prompt = `Write the decomposed rig program for this task.\n\n${taskText}`;
      } else if (prevSource.trim()) {
        prompt = `Fix this rig program. Fix precisely what the error names and preserve what already worked.\n\nTask:\n${taskText}\n\nCurrent source:\n${prevSource}\n\nCaptured error:\n${prevError}\n\nRequired skeleton:\n${SKELETON}`;
      } else {
        prompt = `There is no source to fix: the previous attempt produced nothing. Generate the complete program from scratch.\n\nTask:\n${taskText}\n\nRequired skeleton:\n${SKELETON}`;
      }
      const raw = await call(writer, prompt, { label: `writer-${n}`, timeout: Math.min(6 * 60_000, Math.max(30_000, left())) });
      const rec: Attempt = { attempt: n, generation: "", typecheck: "skipped", execute: "skipped", error: "" };
      attempts.push(rec);
      const src = raw ? stripFence(raw) : "";
      if (!src.trim()) {
        rec.generation = "empty/null";
        prevSource = "";
        prevError = "empty source";
        continue;
      }
      rec.generation = "ok";
      source = src;
      prevSource = src;
      const tc = await runCli(["--typecheck"], src, Math.min(2 * 60_000, left()));
      if (tc.code !== 0 || tc.timedOut) {
        rec.typecheck = tc.timedOut ? "timeout" : "fail";
        rec.error = `${tc.stdout}\n${tc.stderr}`.trim() || "typecheck failed";
        prevError = rec.error;
        continue;
      }
      rec.typecheck = "pass";
      if (left() < 30_000) { rec.execute = "skipped (no time)"; break; }
      const ex = await runCli(["--server"], src, Math.min(5 * 60_000, left()));
      if (ex.code !== 0 || ex.timedOut) {
        rec.execute = ex.timedOut ? "timeout" : "fail";
        rec.error = `${ex.stderr}\n${ex.stdout}`.trim() || "execute failed";
        prevError = rec.error;
        continue;
      }
      try {
        const parsed = JSON.parse(ex.stdout);
        const sol = parsed?.solution ?? parsed?.output?.solution ?? parsed?.result?.solution;
        if (typeof sol === "string" && sol.trim()) {
          rec.execute = "pass";
          decomposedSolution = sol;
          status = "pass";
          break;
        }
        rec.execute = "fail";
        rec.error = `No solution string in stdout: ${ex.stdout.slice(0, 4000)}`;
      } catch (e) {
        rec.execute = "fail";
        rec.error = `Invalid JSON stdout: ${String(e)}\n${ex.stdout.slice(0, 4000)}`;
      }
      prevError = rec.error;
    }
    const decomposedMs = Date.now() - d0;

    const decomposedForGrading = decomposedSolution.trim() || "(Decomposition produced no solution.)";
    let grade: { singleScore: number; decomposedScore: number; winner: string; rationale: string } | null = null;
    if (left() > 30_000) {
      grade = await call(
        grader,
        `${taskText}\n\n=== SOLUTION A (single-call) ===\n${singleSolution || "(empty)"}\n\n=== SOLUTION B (decomposed) ===\n${decomposedForGrading}`,
        { label: "grade", timeout: Math.min(5 * 60_000, Math.max(30_000, left())) },
      );
    }

    return {
      task,
      singleMs,
      decomposedMs,
      singleSolution,
      decomposedSolution,
      attempts,
      status,
      decomposedSource: source,
      grade,
    };
  },
});