[rig-decomposition-bench] Daily rig decomposition benchmark — 2026-10-11 — tie

Both attempts returned an empty (or whitespace-only) s.string result from the decompositionWriter / decompositionFixer agents, so per the hard requirement in the spec ("Treat a null, missing, or whitespace-only writer result as a source-generation failure: never pass it to the fixer or invoke the CLI with it"), neither attempt reached the typecheck or execute stage — both are recorded as skipped.

No typecheck or execute stderr/stdout was captured because the CLI was never invoked for either attempt.

A separate (non-reported, investigative-only) debug run of the same program confirmed the proximate cause: the single-turn s.string-output agents (singleCallSolver, decompositionWriter, decompositionFixer — each maxTurns: 1 with no repair() addon, per the spec's requirement that only the task-picker gets repair()) returned responses that could not be parsed as a single JSON string value ("No JSON value found." / "Expected property name or '}' in JSON at position 2"), causing call() to return null for all three. This is consistent with how rig's s.string output contract expects the model's raw response to itself be valid JSON (a quoted string), and the underlying model for these calls instead returned free-form Markdown/prose at maxTurns: 1 with no parse-repair retry available.

(empty — the singleCallSolver agent call returned null, so singleCallSolution is the empty string)

Decomposition produced no valid solution within the allotted attempts and time budget.

import { agent, configureAgent, copilotEngine, repair, s, workflow } from "rig";

// Configure the Copilot engine in server (stdio) mode.
configureAgent(copilotEngine({ server: true }));

// Overall wall-clock budget for this benchmark run (~25 minutes from module eval).
const DEADLINE = Date.now() + 25 * 60_000;
const remainingMs = () => DEADLINE - Date.now();
const clamp = (desiredMs: number, minMs: number) => Math.max(minMs, Math.min(desiredMs, remainingMs()));

// Agent role: pick one concrete, complicated single-day task suited to multi-agent decomposition.
const taskPicker = agent({
  model: "small",
  maxTurns: 2,
  addons: repair(),
  instructions:
    "Pick one concrete, complicated task a person or team might face in a single day that would " +
    "naturally benefit from being split across sub-agents running different models: multi-source " +
    "research and synthesis, a multi-file coding task with distinct design/implementation/review " +
    "phases, a structured report combining several independent analyses, or a multi-step data " +
    "transformation pipeline. The task must be fully self-contained and solvable from its " +
    "description alone, with no external file or live web access required, and concrete enough to " +
    "grade. Prefer variety across domains from run to run (avoid always picking software or writing " +
    "tasks). Return a title, a domain, a full description, and 3-6 concrete, independently checkable " +
    "success criteria.",
  output: s.object({
    title: s.string,
    domain: s.string,
    description: s.string("The full self-contained task description, solvable with no external access"),
    criteria: s.array(s.string, "3-6 concrete, independently checkable success criteria"),
  }),
});

// Agent role: solve the whole task alone in a single turn, addressing every success criterion.
const singleCallSolver = agent({
  model: "medium",
  maxTurns: 1,
  instructions:
    "Solve the task completely and address every success criterion listed below, by yourself, with " +
    "no delegation and no tools. Return the complete solution verbatim, including Markdown code " +
    "fences when appropriate.",
  output: s.string,
});

const DECOMPOSITION_SKELETON = `import { agent, workflow, s } from "rig";

// Agent role: <role>
const analyze = agent({ model: "small", instructions: "...", output: s.string });

// Workflow role: combines the agents' outputs into the final solution
export default workflow({
  meta: { name: "decomposed-solution", description: "..." },
  body: async ({ call }) => {
    const analysis = await call(analyze, "...");
    return { solution: analysis ?? "" };
  },
});
`;

const WRITER_CONTRACT =
  "You are writing a self-contained rig TypeScript program (the rig skill is NOT available to you " +
  "for reference, so follow this contract exactly). The program must import only from \"rig\". Its " +
  "root export must be a workflow() that takes NO required input, exported via `export default` " +
  "WITHOUT invoking it. A workflow() spec accepts only `meta`, an optional `input`, and `body`; it " +
  "has no `output` or `execute` field — the final `{ solution }` object returned by the program is " +
  "simply whatever `body` returns. (An agent() root would instead declare " +
  "`output: s.object({ solution: s.string })`, but you must use a workflow() root here.) The program " +
  "must split the task across AT LEAST TWO separate agent() definitions, each with its own " +
  "`// Agent role: ...` comment directly above it, each choosing model: \"small\" for simple " +
  "sub-steps and model: \"medium\" or \"large\" for harder sub-steps, coordinated by the workflow's " +
  "body so that the final answer combines their outputs rather than one agent solving everything by " +
  "itself. Use `s.string` as the output schema for every agent whose result is free-form source text " +
  "or another free-form solution fragment; do not make any sub-agent re-emit a large solution wrapped " +
  "inside an object schema. Here is the required shape, abridged to a single agent for brevity — your " +
  "program must still define at least two agents:\n\n" +
  DECOMPOSITION_SKELETON +
  "\n\nReturn ONLY the complete TypeScript source of the program (no explanation, no markdown fences " +
  "around it unless they are stripped before use).";

// Agent role: write a self-contained rig program that decomposes the task across at least two agents.
const decompositionWriter = agent({
  model: "medium",
  maxTurns: 1,
  instructions: WRITER_CONTRACT,
  output: s.string,
});

// Agent role: fix a previously generated decomposition program using a captured error, or regenerate it from scratch.
const decompositionFixer = agent({
  model: "medium",
  maxTurns: 1,
  instructions: WRITER_CONTRACT,
  output: s.string,
});

// Agent role: grade both solutions against the success criteria and pick a winner.
const grader = agent({
  model: "large",
  maxTurns: 1,
  instructions:
    "Score each of the two candidate solutions 0-10 on how completely and correctly it satisfies the " +
    "listed success criteria. Judge strictly on correctness and completeness of content only — never " +
    "on length, verbosity, or which approach (single call vs decomposition) produced the solution. " +
    "Pick a winner of exactly 'single-call', 'decomposed', or 'tie', and explain the comparison.",
  output: s.object({
    singleCallScore: s.number,
    decomposedScore: s.number,
    winner: s.enum("single-call", "decomposed", "tie"),
    rationale: s.string,
  }),
});

function stripFences(text: string): string {
  const trimmed = text.trim();
  const fenceMatch = trimmed.match(/^```(?:ts|typescript)?\s*\n([\s\S]*?)\n```\s*$/);
  return fenceMatch ? fenceMatch[1].trim() : trimmed;
}

interface SubprocessResult {
  status: "pass" | "fail" | "timeout";
  output: string;
}

async function runRigCli(source: string, args: string[], timeoutMs: number): Promise<SubprocessResult> {
  const { spawn } = await import("node:child_process");
  return new Promise<SubprocessResult>((resolvePromise) => {
    const child = spawn("node", ["rig.ts", ...args], {
      cwd: process.cwd(),
      stdio: ["pipe", "pipe", "pipe"],
    });
    let stdout = "";
    let stderr = "";
    let settled = false;
    const timer = setTimeout(() => {
      if (settled) return;
      settled = true;
      child.kill("SIGKILL");
      resolvePromise({ status: "timeout", output: (stdout + stderr).slice(0, 8000) || "(timed out, no output)" });
    }, Math.max(1000, timeoutMs));

    child.stdout.on("data", (chunk: Buffer) => { stdout += chunk.toString(); });
    child.stderr.on("data", (chunk: Buffer) => { stderr += chunk.toString(); });
    child.on("error", (err: Error) => {
      if (settled) return;
      settled = true;
      clearTimeout(timer);
      resolvePromise({ status: "fail", output: `spawn error: ${err.message}` });
    });
    child.on("close", (code: number | null) => {
      if (settled) return;
      settled = true;
      clearTimeout(timer);
      if (code === 0) {
        resolvePromise({ status: "pass", output: stdout });
      } else {
        resolvePromise({ status: "fail", output: (stderr + stdout).slice(0, 8000) });
      }
    });
    child.stdin.write(source);
    child.stdin.end();
  });
}

interface AttemptRecord {
  attempt: number;
  sourceResult: "ok" | "empty";
  typecheck: "pass" | "fail" | "timeout" | "skipped";
  execute: "pass" | "fail" | "timeout" | "skipped";
  typecheckOutput: string;
  executeOutput: string;
  note: string;
}

// Workflow role: orchestrate task selection, single-call solving, decomposed solving, and grading.
export default workflow({
  meta: {
    name: "rig-decomposition-bench",
    description: "Benchmark single-call vs decomposed multi-agent solving on a daily generated task",
    phases: ["Pick task", "Solve single-call", "Solve decomposed", "Grade"],
  },
  body: async ({ call, phase }) => {
    phase("Pick task");
    const task = await call(taskPicker, "Pick today's benchmark task.");

    phase("Solve single-call");
    const singleCallStart = Date.now();
    let singleCallSolution = "";
    if (task && remainingMs() > 15_000) {
      const result = await call(
        singleCallSolver,
        `Task: ${task.title} (${task.domain})\n\n${task.description}\n\nSuccess criteria:\n` +
          task.criteria.map((c: string, i: number) => `${i + 1}. ${c}`).join("\n"),
      );
      singleCallSolution = result ?? "";
    }
    const singleCallDurationMs = Date.now() - singleCallStart;

    phase("Solve decomposed");
    const decompStart = Date.now();
    const attempts: AttemptRecord[] = [];
    let decomposedSolution = "";
    let decomposedSource = "";
    let decomposedPass = false;

    const taskPrompt = task
      ? `Task to decompose:\nTitle: ${task.title}\nDomain: ${task.domain}\nDescription: ${task.description}\n` +
        `Success criteria:\n${task.criteria.map((c: string, i: number) => `${i + 1}. ${c}`).join("\n")}`
      : "No task was generated; invent a reasonable concrete multi-step task and decompose it.";

    let previousSource = "";
    let previousError = "";
    let previousWasEmpty = false;

    for (let attemptNum = 1; attemptNum <= 2 && !decomposedPass; attemptNum++) {
      if (remainingMs() < 30_000) break;

      let rawSource: string | null;
      if (attemptNum === 1) {
        rawSource = await call(decompositionWriter, taskPrompt);
      } else if (previousWasEmpty) {
        rawSource = await call(
          decompositionFixer,
          `${taskPrompt}\n\nThe previous attempt produced no usable source at all (empty or ` +
            "whitespace-only result). There is no source to fix. Generate the complete program from " +
            "scratch following the task and the required skeleton exactly.",
        );
      } else {
        rawSource = await call(
          decompositionFixer,
          `${taskPrompt}\n\nHere is the previously generated program source:\n\n\`\`\`ts\n${previousSource}\n\`\`\`\n\n` +
            `It failed with this exact captured error:\n\n${previousError}\n\n` +
            "Fix precisely what this error names while preserving everything that already worked. Do " +
            "not invent unrelated API edits.",
        );
      }

      const trimmed = (rawSource ?? "").trim();
      if (!trimmed) {
        attempts.push({
          attempt: attemptNum,
          sourceResult: "empty",
          typecheck: "skipped",
          execute: "skipped",
          typecheckOutput: "",
          executeOutput: "",
          note: "Writer/fixer produced no usable source; next attempt (if any) regenerates from scratch.",
        });
        previousWasEmpty = true;
        continue;
      }

      const source = stripFences(trimmed);
      previousSource = source;
      previousWasEmpty = false;

      if (remainingMs() < 20_000) {
        attempts.push({
          attempt: attemptNum,
          sourceResult: "ok",
          typecheck: "skipped",
          execute: "skipped",
          typecheckOutput: "",
          executeOutput: "",
          note: "Skipped verification: insufficient time budget remaining.",
        });
        break;
      }

      const typecheckTimeout = clamp(2 * 60_000, 15_000);
      const typecheckResult = await runRigCli(source, ["--typecheck"], typecheckTimeout);

      if (typecheckResult.status !== "pass") {
        previousError = `typecheck ${typecheckResult.status}:\n${typecheckResult.output}`;
        attempts.push({
          attempt: attemptNum,
          sourceResult: "ok",
          typecheck: typecheckResult.status,
          execute: "skipped",
          typecheckOutput: typecheckResult.output,
          executeOutput: "",
          note:
            attemptNum < 2
              ? "Next attempt asks the fixer to resolve this exact typecheck error."
              : "No further attempts remain.",
        });
        decomposedSource = source;
        continue;
      }

      if (remainingMs() < 15_000) {
        attempts.push({
          attempt: attemptNum,
          sourceResult: "ok",
          typecheck: "pass",
          execute: "skipped",
          typecheckOutput: typecheckResult.output,
          executeOutput: "",
          note: "Skipped execution: insufficient time budget remaining after typecheck.",
        });
        decomposedSource = source;
        break;
      }

      const executeTimeout = clamp(5 * 60_000, 20_000);
      const executeResult = await runRigCli(source, ["--server"], executeTimeout);
      decomposedSource = source;

      if (executeResult.status !== "pass") {
        previousError = `execute ${executeResult.status}:\n${executeResult.output}`;
        attempts.push({
          attempt: attemptNum,
          sourceResult: "ok",
          typecheck: "pass",
          execute: executeResult.status,
          typecheckOutput: typecheckResult.output,
          executeOutput: executeResult.output,
          note:
            attemptNum < 2
              ? "Next attempt asks the fixer to resolve this exact execute error."
              : "No further attempts remain.",
        });
        continue;
      }

      // Execute succeeded; parse the final solution from stdout JSON.
      let parsedSolution = "";
      try {
        const parsed = JSON.parse(executeResult.output);
        if (parsed && typeof parsed === "object" && typeof parsed.solution === "string") {
          parsedSolution = parsed.solution;
        } else if (typeof parsed === "string") {
          parsedSolution = parsed;
        } else {
          parsedSolution = executeResult.output;
        }
      } catch {
        parsedSolution = executeResult.output;
      }

      decomposedSolution = parsedSolution;
      decomposedPass = true;
      attempts.push({
        attempt: attemptNum,
        sourceResult: "ok",
        typecheck: "pass",
        execute: "pass",
        typecheckOutput: typecheckResult.output,
        executeOutput: executeResult.output,
        note: "Succeeded; no further attempts needed.",
      });
    }

    const decompDurationMs = Date.now() - decompStart;

    phase("Grade");
    const placeholderSolution =
      "Decomposition produced no valid solution within the allotted attempts and time budget.";
    let grading: Awaited<ReturnType<typeof grader>> | null = null;
    if (remainingMs() > 15_000) {
      grading = await call(
        grader,
        `${taskPrompt}\n\n` +
          `--- Single-call solution ---\n${singleCallSolution || "(no solution produced)"}\n\n` +
          `--- Decomposed solution ---\n${decomposedPass ? decomposedSolution : placeholderSolution}`,
      );
    }

    return {
      task,
      singleCallDurationMs,
      decompDurationMs,
      singleCallSolution,
      decomposedSolution: decomposedPass ? decomposedSolution : placeholderSolution,
      decomposedPass,
      attempts,
      grading,
      decomposedSource,
    };
  },
});