[rig-decomposition-bench] Daily rig decomposition benchmark — 2026-10-15 — no result (task picker failed)

import { agent, workflow, repair, configureAgent, copilotEngine, s } from "rig";

configureAgent(copilotEngine());

const DEADLINE = Date.now() + 25 * 60_000;
const left = () => DEADLINE - Date.now();

type RunResult = { status: "passed" | "failed" | "skipped"; stdout: string; stderr: string };

const cp = (globalThis as any).process.getBuiltinModule("node:child_process");

function runRig(source: string, flag: "--typecheck" | "--server", maxMs: number): Promise<RunResult> {
  const budget = Math.min(maxMs, left());
  if (budget < 5_000) return Promise.resolve({ status: "skipped", stdout: "", stderr: "time budget exhausted" });
  return new Promise((resolve) => {
    const child = cp.spawn("node", ["rig.ts", flag], { cwd: (globalThis as any).process.cwd(), stdio: ["pipe", "pipe", "pipe"] });
    let out = "";
    let err = "";
    let timedOut = false;
    const timer = setTimeout(() => { timedOut = true; child.kill("SIGKILL"); }, budget);
    child.stdout.on("data", (d: any) => { out += String(d); });
    child.stderr.on("data", (d: any) => { err += String(d); });
    child.on("error", (e: any) => { clearTimeout(timer); resolve({ status: "failed", stdout: out, stderr: err + String(e) }); });
    child.on("close", (code: number | null) => {
      clearTimeout(timer);
      if (timedOut) err += `\nTimed out after ${budget} ms`;
      resolve({ status: code === 0 && !timedOut ? "passed" : "failed", stdout: out, stderr: err });
    });
    child.stdin.on("error", () => {});
    child.stdin.end(source);
  });
}

// Agent role: pick a concrete benchmark task
const pickTask = agent({
  model: "small",
  maxTurns: 2,
  addons: repair(),
  instructions:
    "Pick one concrete, complicated task a person or team might face in a single day that would naturally benefit from being split across sub-agents running different models: multi-source research and synthesis, a multi-file coding task with design/implementation/review phases, a structured report combining several independent analyses, or a multi-step data transformation pipeline. It must be self-contained (no external files or live web) and gradable. Vary the domain; today's seed: " +
    String(Date.now() % 100000) +
    ". Return a title, a domain, a description, and 3-6 concrete checkable success criteria.",
  output: s.object({
    title: s.string,
    domain: s.string,
    description: s.string,
    criteria: s.array(s.string),
  }),
});

// Agent role: solve the whole task in one call
const solveSingle = agent({
  model: "medium",
  maxTurns: 1,
  instructions: "Solve the whole task yourself in one response, addressing every success criterion. No delegation, no tools. Return the complete solution.",
  output: s.string,
});

// Agent role: write the decomposed rig program
const writeProgram = agent({
  model: "medium",
  maxTurns: 1,
  instructions: "You write self-contained rig TypeScript programs. Return only the full TypeScript source, no commentary (a single ```ts fence is fine).",
  output: s.string,
});

// Agent role: fix or generate the rig program
const fixProgram = agent({
  model: "medium",
  maxTurns: 1,
  instructions: "You fix or generate self-contained rig TypeScript programs. Return only the full TypeScript source, no commentary (a single ```ts fence is fine).",
  output: s.string,
});

// Agent role: grade both solutions
const grader = agent({
  model: "large",
  maxTurns: 1,
  instructions:
    "Score each solution 0-10 on how completely and correctly it satisfies the success criteria. Judge only on correctness and completeness of content — not length and not which approach produced it. Pick a winner (single-call, decomposed, or tie) and explain the comparison.",
  output: s.object({
    singleScore: s.number,
    decomposedScore: s.number,
    winner: s.enum("single-call", "decomposed", "tie"),
    rationale: s.string,
  }),
});

const SKELETON = `import { agent, workflow, s } from "rig";

// Agent role: <role>
const analyze = agent({ model: "small", instructions: "...", output: s.string });

// Workflow role: combines the agents' outputs into the final solution
export default workflow({
  meta: { name: "decomposed-solution", description: "..." },
  body: async ({ call }) => {
    const analysis = await call(analyze, "...");
    return { solution: analysis ?? "" };
  },
});`;

const CONTRACT = `Rules: import only from "rig". Use at least two agents (the skeleton is abridged to one), each preceded by a "// Agent role: ..." comment, each with model "small" for simple sub-steps and "medium" or "large" for harder ones. Coordinate them so the final answer combines their outputs. The root takes no required input and is exported with \`export default\` without being invoked. A workflow() spec accepts only meta, an optional input, and body; it has no output or execute field — the final { solution } object is what body returns. An agent() root instead declares output: s.object({ solution: s.string }). Use s.string for every agent whose result is source text or free-form solution; do not make agents re-emit large solutions inside object wrappers. Do not use console.log.
Required shape:
${SKELETON}`;

function stripFence(src: string | null): string {
  const t = (src ?? "").trim();
  const m = t.match(/```(?:ts|typescript|rig)?\n([\s\S]*?)```/);
  return (m ? m[1] : t).trim();
}

type Attempt = {
  attempt: number;
  generation: string;
  typecheck: string;
  execute: string;
  error: string;
  note: string;
};

export default workflow({
  meta: { name: "rig-decomposition-bench", description: "Benchmarks single-call vs decomposed solving" },
  body: async ({ call }) => {
    const task = await call(pickTask, "Pick the task.");
    if (!task) return { error: "task picker returned null" };
    const taskText = `Title: ${task.title}\nDomain: ${task.domain}\nDescription: ${task.description}\nSuccess criteria:\n${task.criteria.map((c: string) => `- ${c}`).join("\n")}`;

    const t0 = Date.now();
    const single = (await call(solveSingle, taskText)) ?? "";
    const singleMs = Date.now() - t0;

    const t1 = Date.now();
    const attempts: Attempt[] = [];
    let source = "";
    let solution = "";
    let status: "pass" | "fail" = "fail";
    let lastError = "";
    let lastSource = "";
    for (let i = 1; i <= 2; i++) {
      if (left() < 4 * 60_000) break;
      let raw: string | null;
      if (i === 1) {
        raw = await call(writeProgram, `Write a rig program that solves this task by decomposing it across multiple agents.\n\n${taskText}\n\n${CONTRACT}`);
      } else if (lastSource) {
        raw = await call(fixProgram, `Fix this rig program precisely according to the error, preserving what already worked.\n\nTask:\n${taskText}\n\nSource:\n${lastSource}\n\nExact error:\n${lastError}\n\n${CONTRACT}`);
      } else {
        raw = await call(fixProgram, `There is no source to fix: the previous attempt produced nothing. Generate the complete program from scratch.\n\nTask:\n${taskText}\n\n${CONTRACT}`);
      }
      const src = stripFence(raw);
      const rec: Attempt = { attempt: i, generation: src ? "non-empty" : "empty", typecheck: "skipped", execute: "skipped", error: "", note: "" };
      attempts.push(rec);
      if (!src) {
        lastError = "writer returned empty source";
        rec.error = lastError;
        continue;
      }
      lastSource = src;
      source = src;
      const tc = await runRig(src, "--typecheck", 2 * 60_000);
      rec.typecheck = tc.status;
      if (tc.status !== "passed") {
        lastError = `typecheck:\n${tc.stdout}\n${tc.stderr}`.trim();
        rec.error = lastError;
        rec.note = i < 2 ? "Passed source and typecheck error to fixer" : "";
        continue;
      }
      const ex = await runRig(src, "--server", 5 * 60_000);
      rec.execute = ex.status;
      if (ex.status !== "passed") {
        lastError = `execute:\n${ex.stdout.slice(-3000)}\n${ex.stderr.slice(-6000)}`.trim();
        rec.error = lastError;
        rec.note = i < 2 ? "Passed source and execute error to fixer" : "";
        continue;
      }
      try {
        const parsed = JSON.parse(ex.stdout.trim());
        solution = typeof parsed === "string" ? parsed : String(parsed?.solution ?? "");
      } catch {
        solution = ex.stdout.trim();
      }
      if (solution.trim()) {
        status = "pass";
        break;
      }
      lastError = "execution produced empty solution\n" + ex.stdout;
      rec.error = lastError;
    }
    const decomposedMs = Date.now() - t1;

    const decomposedForGrade = solution.trim() ? solution : "(decomposition produced no solution)";
    const grade = await call(
      grader,
      `${taskText}\n\n=== SOLUTION A (single-call) ===\n${single || "(empty)"}\n\n=== SOLUTION B (decomposed) ===\n${decomposedForGrade}`,
    );

    return { task, singleMs, decomposedMs, singleSolution: single, decomposedSolution: solution, attempts, status, grade, decomposedSource: source };
  },
});