[rig-decomposition-bench] Daily rig decomposition benchmark — 2026-10-13 — tie (both failed)

no source produced

no source produced

The single-call solver (medium model, 1 turn, s.string output) returned an empty string. No solution content was produced to display.

(no solution: decomposition did not produce a successful run)

import { agent, configureAgent, copilotEngine, repair, s, workflow } from "rig";
import { execFile } from "node:child_process";

// Configure the Copilot engine in server (stdio) mode for this benchmark run.
configureAgent(copilotEngine({ server: true }));

const DEADLINE_MS = Date.now() + 25 * 60_000;
function remaining(): number {
  return DEADLINE_MS - Date.now();
}
function clamp(ms: number, max: number): number {
  return Math.max(0, Math.min(ms, max));
}

const RIG_CLI = `${process.cwd()}/rig.ts`;

type RunResult = { stdout: string; stderr: string; code: number | null; timedOut: boolean };

function runRigCli(args: string[], stdin: string, timeoutMs: number): Promise<RunResult> {
  return new Promise((resolve) => {
    if (timeoutMs <= 0) {
      resolve({ stdout: "", stderr: "skipped: no time budget remaining", code: null, timedOut: true });
      return;
    }
    const child = execFile(
      "node",
      [RIG_CLI, ...args],
      { cwd: process.cwd(), timeout: timeoutMs, maxBuffer: 1024 * 1024 * 50 },
      (error, stdout, stderr) => {
        const timedOut = !!error && (error as any).signal === "SIGTERM";
        resolve({ stdout, stderr, code: error ? (error as any).code ?? null : 0, timedOut });
      },
    );
    child.stdin?.write(stdin);
    child.stdin?.end();
  });
}

// Task picker schema
const TaskSchema = s.object({
  title: s.string,
  domain: s.string,
  description: s.string,
  successCriteria: s.nonEmptyArray(s.string),
});

// Agent role: pick a concrete, self-contained daily task suited to multi-agent decomposition.
const taskPicker = agent({
  name: "taskPicker",
  model: "small",
  maxTurns: 2,
  addons: repair(),
  instructions:
    "Pick one concrete, complicated task that a person or team might face in a single day, " +
    "which would naturally benefit from being split across sub-agents running different models " +
    "(e.g. multi-source research and synthesis, a multi-file coding task with distinct " +
    "design/implementation/review phases, a structured report combining several independent " +
    "analyses, or a multi-step data transformation pipeline). The task must be fully " +
    "self-contained and solvable purely from its own description, with no external file or " +
    "live web access required. Prefer variety across domains. Return a title, a domain, a " +
    "description, and 3-6 concrete, checkable success criteria.",
  output: TaskSchema,
});

// Agent role: solve the whole task alone in a single turn, addressing every success criterion.
const soloSolver = agent({
  name: "soloSolver",
  model: "medium",
  maxTurns: 1,
  instructions:
    "You are given a task description and its success criteria. Produce the complete solution " +
    "to the task by yourself in this single response, addressing every success criterion. " +
    "Do not delegate, do not call any tools. Return only the solution content (including " +
    "Markdown code fences if appropriate) -- do not wrap it in JSON or add commentary about " +
    "what you are doing.",
  output: s.string,
});

// Agent role: write a second, self-contained rig program that decomposes the task across
// at least two agents of varying model sizes, combining their outputs into a final solution.
const decompositionWriter = agent({
  name: "decompositionWriter",
  model: "medium",
  maxTurns: 1,
  instructions:
    "Write a complete, self-contained TypeScript program using the 'rig' agent framework " +
    "that solves the given task by splitting it across at least two agents, each with a " +
    "`// Agent role: ...` comment above it, choosing `small` for simple sub-steps and " +
    "`medium` or `large` for harder ones. The agents must be coordinated so the final answer " +
    "combines their outputs rather than one agent solving everything alone. The program must " +
    "import only from \"rig\" (never any other package), must take no required input on its " +
    "root, and must export its root via `export default` without invoking it.\n\n" +
    "You do not have access to the rig skill documentation, so follow this exact root " +
    "contract: a `workflow()` spec accepts only `meta`, an optional `input`, and `body`; it " +
    "has no `output` or `execute` field, and the final `{ solution }` object is simply what " +
    "`body` returns. An `agent()` root instead declares " +
    "`output: s.object({ solution: s.string })`. Use `s.string` for every agent whose result " +
    "is source text or another free-form solution; reserve object schemas for structured " +
    "metadata, and never make a solver re-emit a large solution inside a wrapper object.\n\n" +
    "Here is the required skeleton shape (abridged to one agent -- your program must still " +
    "define at least two agents):\n\n" +
    "```ts\n" +
    "import { agent, workflow, s } from \"rig\";\n\n" +
    "// Agent role: <role>\n" +
    "const analyze = agent({ model: \"small\", instructions: \"...\", output: s.string });\n\n" +
    "// Workflow role: combines the agents' outputs into the final solution\n" +
    "export default workflow({\n" +
    "  meta: { name: \"decomposed-solution\", description: \"...\" },\n" +
    "  body: async ({ call }) => {\n" +
    "    const analysis = await call(analyze, \"...\");\n" +
    "    return { solution: analysis ?? \"\" };\n" +
    "  },\n" +
    "});\n" +
    "```\n\n" +
    "Return ONLY the raw TypeScript source code of the program, with no Markdown code fences " +
    "and no commentary before or after it.",
  output: s.string,
});

// Agent role: fix a previously generated decomposition program given its exact captured error.
const decompositionFixer = agent({
  name: "decompositionFixer",
  model: "medium",
  maxTurns: 1,
  instructions:
    "You previously wrote (or are now writing for the first time) a self-contained TypeScript " +
    "program using the 'rig' agent framework to solve a given task by splitting it across at " +
    "least two agents, each with a `// Agent role: ...` comment above it, choosing `small` for " +
    "simple sub-steps and `medium` or `large` for harder ones, coordinated so the final answer " +
    "combines their outputs rather than one agent solving everything. The program must import " +
    "only from \"rig\", must take no required input on its root, and must export its root via " +
    "`export default` without invoking it.\n\n" +
    "Required root contract: a `workflow()` spec accepts only `meta`, an optional `input`, and " +
    "`body`; it has no `output` or `execute` field, and the final `{ solution }` object is " +
    "simply what `body` returns. An `agent()` root instead declares " +
    "`output: s.object({ solution: s.string })`. Use `s.string` for every agent whose result " +
    "is source text or another free-form solution; reserve object schemas for structured " +
    "metadata, and never make a solver re-emit a large solution inside a wrapper object.\n\n" +
    "Required skeleton shape (abridged to one agent -- the full program must still define at " +
    "least two agents):\n\n" +
    "```ts\n" +
    "import { agent, workflow, s } from \"rig\";\n\n" +
    "// Agent role: <role>\n" +
    "const analyze = agent({ model: \"small\", instructions: \"...\", output: s.string });\n\n" +
    "// Workflow role: combines the agents' outputs into the final solution\n" +
    "export default workflow({\n" +
    "  meta: { name: \"decomposed-solution\", description: \"...\" },\n" +
    "  body: async ({ call }) => {\n" +
    "    const analysis = await call(analyze, \"...\");\n" +
    "    return { solution: analysis ?? \"\" };\n" +
    "  },\n" +
    "});\n" +
    "```\n\n" +
    "If prior source is provided below, fix precisely what the captured error names while " +
    "preserving everything that already worked -- do not invent unrelated API edits. If no " +
    "prior source is provided, generate the complete program from scratch using the task and " +
    "skeleton above.\n\n" +
    "Return ONLY the raw TypeScript source code of the program, with no Markdown code fences " +
    "and no commentary before or after it.",
  output: s.string,
});

// Agent role: grade both solutions for correctness/completeness only, and pick a winner.
const grader = agent({
  name: "grader",
  model: "large",
  maxTurns: 1,
  instructions:
    "You are given a task with success criteria, and two candidate solutions: 'single-call' " +
    "and 'decomposed'. Score each solution from 0 to 10 on how completely and correctly it " +
    "satisfies the success criteria. Judge ONLY correctness and completeness of content -- " +
    "never on length, verbosity, or which approach (single-call vs decomposed) produced it. " +
    "If a solution is an explicit placeholder stating that no solution was produced, score it " +
    "low and explain why. Pick a winner: 'single-call', 'decomposed', or 'tie'. Explain your " +
    "comparison rationale referencing the success criteria.",
  output: s.object({
    singleCallScore: s.number,
    decomposedScore: s.number,
    winner: s.enum("single-call", "decomposed", "tie"),
    rationale: s.string,
  }),
});

function stripFences(src: string): string {
  const trimmed = src.trim();
  const fenceMatch = trimmed.match(/^```(?:ts|typescript)?\n([\s\S]*?)\n```$/);
  return fenceMatch ? fenceMatch[1] : trimmed;
}

type AttemptRecord = {
  attempt: number;
  sourceGenerated: boolean;
  sourceLength: number;
  typecheck: { status: "passed" | "failed" | "skipped"; output: string };
  execute: { status: "passed" | "failed" | "skipped"; output: string };
  note: string;
};

// Workflow role: orchestrate the rig decomposition benchmark end to end within a wall-clock budget.
export default workflow({
  meta: {
    name: "rig-decomposition-bench",
    description: "Compare a single-call solve against a multi-agent decomposed solve for one generated task.",
  },
  body: async ({ call }) => {
    const attempts: AttemptRecord[] = [];
    let decomposedSource = "";
    let decomposedSolution = "";
    let decomposedStatus: "pass" | "fail" = "fail";

    // Step 1: pick a task.
    const task = remaining() > 30_000
      ? await call(taskPicker, "Pick the task now.")
      : null;

    if (!task) {
      return {
        task: null,
        singleCallDurationMs: 0,
        decomposedDurationMs: 0,
        singleCallSolution: "",
        decomposedSolution: "",
        decomposedStatus: "fail" as const,
        attempts,
        grading: null,
        generatedProgramSource: "",
        failureReason: "Task picker failed or ran out of time budget before starting.",
      };
    }

    const taskPrompt =
      `Task: ${task.title}\nDomain: ${task.domain}\nDescription: ${task.description}\n` +
      `Success criteria:\n${task.successCriteria.map((c, i) => `${i + 1}. ${c}`).join("\n")}`;

    // Step 2: solve it in one call.
    const singleCallStart = Date.now();
    const singleCallSolution = remaining() > 30_000
      ? (await call(soloSolver, taskPrompt)) ?? ""
      : "";
    const singleCallDurationMs = Date.now() - singleCallStart;

    // Step 3: solve it by decomposition.
    const decompStart = Date.now();
    let priorSource: string | null = null;
    let priorError: string | null = null;

    for (let attemptNum = 1; attemptNum <= 2; attemptNum++) {
      if (remaining() < 60_000) {
        attempts.push({
          attempt: attemptNum,
          sourceGenerated: false,
          sourceLength: 0,
          typecheck: { status: "skipped", output: "insufficient time budget remaining" },
          execute: { status: "skipped", output: "insufficient time budget remaining" },
          note: "skipped: not enough wall-clock budget left",
        });
        break;
      }

      let rawSource: string | null;
      if (attemptNum === 1) {
        rawSource = await call(decompositionWriter, taskPrompt);
      } else {
        const fixerPrompt: string = priorSource
          ? `${taskPrompt}\n\nPrior generated source:\n${priorSource}\n\nCaptured error to fix:\n${priorError}`
          : `${taskPrompt}\n\nThere is no prior source (the first attempt produced nothing). Generate the complete program from scratch.`;
        rawSource = await call(decompositionFixer, fixerPrompt);
      }

      const sourceGenerated: boolean = !!rawSource && rawSource.trim().length > 0;
      const source: string = sourceGenerated ? stripFences(rawSource as string) : "";

      if (!sourceGenerated) {
        attempts.push({
          attempt: attemptNum,
          sourceGenerated: false,
          sourceLength: 0,
          typecheck: { status: "skipped", output: "no source produced" },
          execute: { status: "skipped", output: "no source produced" },
          note: attemptNum === 1 ? "empty result; next attempt asked to generate from scratch" : "empty result on final attempt",
        });
        priorSource = null;
        priorError = "No source was generated by the writer/fixer.";
        continue;
      }

      // Typecheck
      const typecheckTimeout = clamp(remaining() - 30_000, 2 * 60_000);
      const tc = await runRigCli(["--typecheck"], source, typecheckTimeout);
      const typecheckOutput = `${tc.stdout}\n${tc.stderr}`.trim();
      const typecheckPassed = tc.code === 0 && /typecheck passed/i.test(typecheckOutput);

      if (!typecheckPassed) {
        attempts.push({
          attempt: attemptNum,
          sourceGenerated: true,
          sourceLength: source.length,
          typecheck: { status: tc.timedOut ? "skipped" : "failed", output: typecheckOutput || "(no output)" },
          execute: { status: "skipped", output: "skipped: typecheck did not pass" },
          note: attemptNum === 1 ? "typecheck failed; next attempt asked to fix the exact error" : "typecheck failed on final attempt",
        });
        priorSource = source;
        priorError = typecheckOutput;
        continue;
      }

      // Execute
      const executeTimeout = clamp(remaining() - 10_000, 5 * 60_000);
      const ex = await runRigCli(["--server"], source, executeTimeout);
      const executeOutput = `${ex.stdout}\n${ex.stderr}`.trim();
      let parsedSolution: string | null = null;
      try {
        const parsed = JSON.parse(ex.stdout.trim());
        if (typeof parsed === "string") {
          parsedSolution = parsed;
        } else if (parsed && typeof parsed === "object" && typeof (parsed as any).solution === "string") {
          parsedSolution = (parsed as any).solution;
        } else if (parsed && typeof parsed === "object" && typeof (parsed as any).text === "string") {
          parsedSolution = (parsed as any).text;
        }
      } catch {
        if (ex.code === 0 && ex.stdout.trim().length > 0) {
          parsedSolution = ex.stdout.trim();
        }
      }

      const executePassed = ex.code === 0 && !ex.timedOut && parsedSolution !== null;

      attempts.push({
        attempt: attemptNum,
        sourceGenerated: true,
        sourceLength: source.length,
        typecheck: { status: "passed", output: typecheckOutput || "(no output)" },
        execute: { status: executePassed ? "passed" : "failed", output: executeOutput || "(no output)" },
        note: executePassed
          ? "succeeded"
          : attemptNum === 1
          ? "execute failed; next attempt asked to fix the exact error"
          : "execute failed on final attempt",
      });

      if (executePassed) {
        decomposedSource = source;
        decomposedSolution = parsedSolution ?? "";
        decomposedStatus = "pass";
        break;
      } else {
        priorSource = source;
        priorError = executeOutput;
      }
    }

    if (decomposedStatus !== "pass" && attempts.length > 0) {
      const last = attempts[attempts.length - 1];
      if (last.sourceGenerated && last.typecheck.status === "passed") {
        decomposedSource = decomposedSource || "";
      }
    }

    const decomposedDurationMs = Date.now() - decompStart;

    const finalDecomposedSolution = decomposedStatus === "pass"
      ? decomposedSolution
      : "(no solution: decomposition did not produce a successful run)";

    // Step 4: grade both.
    const grading = remaining() > 30_000
      ? await call(
          grader,
          `${taskPrompt}\n\nSolution A (single-call):\n${singleCallSolution || "(empty)"}\n\n` +
            `Solution B (decomposed):\n${finalDecomposedSolution}`,
        )
      : null;

    return {
      task,
      singleCallDurationMs,
      decomposedDurationMs,
      singleCallSolution,
      decomposedSolution: finalDecomposedSolution,
      decomposedStatus,
      attempts,
      grading,
      generatedProgramSource: decomposedSource,
    };
  },
});