[rig-decomposition-bench] Daily rig decomposition benchmark — 2026-10-28 — no result (task picker failed)

import { agent, workflow, s, repair, configureAgent, copilotEngine } from "rig";

configureAgent(copilotEngine());

const DEADLINE = Date.now() + 25 * 60_000;
const left = () => DEADLINE - Date.now();

const SKELETON = `import { agent, workflow, s } from "rig";

// Agent role: <role>
const analyze = agent({ model: "small", instructions: "...", output: s.string });

// Workflow role: combines the agents' outputs into the final solution
export default workflow({
  meta: { name: "decomposed-solution", description: "..." },
  body: async ({ call }) => {
    const analysis = await call(analyze, "...");
    return { solution: analysis ?? "" };
  },
});`;

const RULES = `Rules for the program (rig API, imports ONLY from "rig"):
- Import only from "rig". No required input on the root. Export the root via \`export default\` without invoking it. No console.log.
- Use at least two agents, each preceded by a \`// Agent role: ...\` comment; choose model "small" for simple sub-steps and "medium" or "large" for harder ones.
- The final answer must combine the agents' outputs, not come from one agent solving everything.
- A workflow() spec accepts ONLY meta, an optional input, and body. It has no output or execute field; the final { solution } object is simply what body returns. An agent() root instead declares output: s.object({ solution: s.string }).
- Agents are called inside body with \`await call(agent, "prompt string")\`, which returns the output or null on failure.
- Use s.string for every agent whose result is source text or free-form solution. Do not wrap large solutions in object schemas.
- Required shape (abridged to one agent; you need at least two):
\`\`\`ts
${SKELETON}
\`\`\`
Return ONLY the complete TypeScript source (no commentary).`;

const taskSchema = s.object({
  title: s.string,
  domain: s.string,
  description: s.string,
  criteria: s.array(s.string),
});

// Agent role: pick a complicated, gradeable task suited to multi-agent decomposition.
const picker = agent({
  model: "small",
  maxTurns: 2,
  addons: [repair()],
  instructions:
    "Pick ONE concrete, complicated task a person or team might face in a single day that would naturally benefit from being split across sub-agents running different models: multi-source research and synthesis, a multi-file coding task with design/implementation/review phases, a structured report combining independent analyses, or a multi-step data transformation pipeline. It must be self-contained (no external files or live web access) and concrete enough to grade. Vary the domain from run to run (pick randomly). Return title, domain, description, and 3-6 concrete checkable success criteria.",
  output: taskSchema,
});

// Agent role: solve the whole task alone in one turn.
const single = agent({
  model: "medium",
  maxTurns: 1,
  instructions:
    "Solve the whole task yourself, in a single response, addressing every success criterion. No delegation, no tools. Return the complete solution verbatim (Markdown code fences allowed).",
  output: s.string,
});

// Agent role: write or fix a rig program that decomposes the task.
const writer = agent({
  model: "medium",
  maxTurns: 1,
  instructions: "You write self-contained rig TypeScript programs. Follow the stated rules exactly.",
  output: s.string,
});

const gradeSchema = s.object({
  singleScore: s.number,
  decomposedScore: s.number,
  winner: s.enum("single-call", "decomposed", "tie"),
  rationale: s.string,
});

// Agent role: grade both solutions on content correctness and completeness only.
const grader = agent({
  model: "large",
  maxTurns: 1,
  instructions:
    "Score each solution 0-10 on how completely and correctly it satisfies the success criteria. Judge only correctness and completeness of content: not length, and not which approach produced it. Pick winner single-call, decomposed, or tie, and explain the comparison.",
  output: gradeSchema,
});

type Run = { code: number | null; stdout: string; stderr: string; timedOut: boolean };

async function runCli(args: string[], src: string, timeoutMs: number): Promise<Run> {
  const { spawn } = await import("node:child_process");
  return new Promise((resolve) => {
    const child = spawn("node", ["rig.ts", ...args], { cwd: process.cwd(), stdio: ["pipe", "pipe", "pipe"] });
    let stdout = "";
    let stderr = "";
    let timedOut = false;
    const timer = setTimeout(() => {
      timedOut = true;
      child.kill("SIGKILL");
    }, timeoutMs);
    child.stdout.on("data", (d: unknown) => { stdout += String(d); });
    child.stderr.on("data", (d: unknown) => { stderr += String(d); });
    child.on("error", (e: Error) => {
      clearTimeout(timer);
      resolve({ code: -1, stdout, stderr: stderr + String(e), timedOut });
    });
    child.on("close", (code: number | null) => {
      clearTimeout(timer);
      resolve({ code, stdout, stderr, timedOut });
    });
    child.stdin.on("error", () => {});
    child.stdin.end(src);
  });
}

type Attempt = {
  attempt: number;
  generation: string;
  typecheck: string;
  execute: string;
  error: string;
};

function stripFence(t: string): string {
  const m = t.match(/```(?:ts|typescript|rig)?\s*\n([\s\S]*?)```/);
  return (m ? m[1] : t).trim();
}

export default workflow({
  meta: { name: "rig-decomposition-bench", description: "Compare single-call vs decomposed solving", phases: ["Pick", "Single", "Decompose", "Grade"] },
  body: async ({ call, phase }) => {
    phase("Pick");
    const task = await call(picker, "Pick a task now.");
    if (!task) return { error: "task picker returned null", status: "fail" };
    const taskText = `Title: ${task.title}\nDomain: ${task.domain}\nDescription: ${task.description}\nSuccess criteria:\n${task.criteria.map((c: string) => `- ${c}`).join("\n")}`;

    phase("Single");
    const t0 = Date.now();
    const singleSolution = (await call(single, taskText)) ?? "";
    const singleMs = Date.now() - t0;

    phase("Decompose");
    const t1 = Date.now();
    const attempts: Attempt[] = [];
    let source = "";
    let decomposedSolution = "";
    let status = "fail";
    let lastError = "";
    let prevSource = "";
    for (let i = 1; i <= 2; i++) {
      if (left() < 4 * 60_000) break;
      const rec: Attempt = { attempt: i, generation: "", typecheck: "skipped", execute: "skipped", error: "" };
      attempts.push(rec);
      let prompt: string;
      if (i === 1) {
        prompt = `Write a rig program that solves this task by decomposing it across at least two agents.\n\nTASK:\n${taskText}\n\n${RULES}`;
      } else if (prevSource.trim()) {
        prompt = `Fix this rig program. Fix precisely what the error names and preserve what already worked; do not make unrelated API edits.\n\nTASK:\n${taskText}\n\nERROR:\n${lastError}\n\nSOURCE:\n${prevSource}\n\n${RULES}`;
      } else {
        prompt = `There is no source to fix (the previous attempt produced nothing). Generate the complete program from scratch.\n\nTASK:\n${taskText}\n\n${RULES}`;
      }
      const raw = await call(writer, prompt);
      const src = raw && raw.trim() ? stripFence(raw) : "";
      if (!src) {
        rec.generation = "empty";
        rec.error = "writer returned null/empty";
        lastError = rec.error;
        prevSource = "";
        continue;
      }
      rec.generation = "ok";
      source = src;
      prevSource = src;
      const tcMs = Math.min(2 * 60_000, left());
      if (tcMs < 10_000) { rec.error = "time budget exhausted"; break; }
      const tc = await runCli(["--typecheck"], src, tcMs);
      if (tc.code !== 0) {
        rec.typecheck = tc.timedOut ? "timeout" : "fail";
        rec.error = (tc.stdout + tc.stderr).trim();
        lastError = rec.error;
        continue;
      }
      rec.typecheck = "pass";
      const exMs = Math.min(5 * 60_000, left());
      if (exMs < 10_000) { rec.execute = "skipped"; rec.error = "time budget exhausted"; break; }
      const ex = await runCli(["--server"], src, exMs);
      if (ex.code !== 0) {
        rec.execute = ex.timedOut ? "timeout" : "fail";
        rec.error = (ex.stdout + ex.stderr).trim();
        lastError = rec.error;
        continue;
      }
      try {
        const parsed = JSON.parse(ex.stdout.trim()) as { solution?: unknown };
        if (typeof parsed.solution === "string" && parsed.solution.trim()) {
          decomposedSolution = parsed.solution;
          rec.execute = "pass";
          status = "pass";
          break;
        }
        rec.execute = "fail";
        rec.error = "no solution string in stdout: " + ex.stdout.slice(0, 2000);
      } catch (e) {
        rec.execute = "fail";
        rec.error = "stdout not valid JSON: " + ex.stdout.slice(0, 2000);
      }
      lastError = rec.error;
    }
    const decomposedMs = Date.now() - t1;

    phase("Grade");
    const placeholder = "(decomposition produced no solution)";
    const grade = await call(
      grader,
      `${taskText}\n\n=== SOLUTION A (single-call) ===\n${singleSolution || "(empty)"}\n\n=== SOLUTION B (decomposed) ===\n${decomposedSolution || placeholder}`,
    );
    return { task, singleMs, decomposedMs, singleSolution, decomposedSolution, attempts, status, grade, decomposedSource: source };
  },
});