[rig-decomposition-bench] Daily rig decomposition benchmark — 2026-10-16 — tie

Error: spawn node ENOENT

Error: spawn node ENOENT

(empty — the solver returned null)

(empty — no program executed successfully)

import { agent, workflow, s, configureAgent, copilotEngine, repair } from "rig";

configureAgent(copilotEngine({ server: true } as never));

const DEADLINE = Date.now() + 25 * 60_000;
const remaining = () => DEADLINE - Date.now();

type RunResult = { ok: boolean; stdout: string; stderr: string; status: string };

async function runRig(source: string, flag: "--typecheck" | "--server", capMs: number): Promise<RunResult> {
  const left = remaining();
  const timeoutMs = Math.min(capMs, left);
  if (timeoutMs < 5_000) return { ok: false, stdout: "", stderr: "skipped: time budget exhausted", status: "skipped" };
  const { spawn } = await import("node:child_process");
  const path = await import("node:path");
  const cwd = path.resolve(process.cwd(), ".github/skills/rig");
  return new Promise<RunResult>((resolve) => {
    const child = spawn("node", ["rig.ts", flag], { cwd });
    let stdout = "";
    let stderr = "";
    let timedOut = false;
    const timer = setTimeout(() => {
      timedOut = true;
      child.kill("SIGKILL");
    }, timeoutMs);
    child.stdout.on("data", (d: Buffer) => { stdout += d.toString(); });
    child.stderr.on("data", (d: Buffer) => { stderr += d.toString(); });
    child.on("error", (e: Error) => {
      clearTimeout(timer);
      resolve({ ok: false, stdout, stderr: stderr + String(e), status: "failed" });
    });
    child.on("close", (code: number | null) => {
      clearTimeout(timer);
      if (timedOut) resolve({ ok: false, stdout, stderr: stderr + `\ntimed out after ${timeoutMs}ms`, status: "timeout" });
      else resolve({ ok: code === 0, stdout, stderr, status: code === 0 ? "passed" : "failed" });
    });
    child.stdin.on("error", () => {});
    child.stdin.end(source);
  });
}

function truncate(text: string, max = 6000): string {
  return text.length > max ? text.slice(0, max) + `\n...[truncated ${text.length - max} chars]` : text;
}

// Agent role: picks a benchmark task
const pickTask = agent({
  name: "taskPicker",
  model: "small",
  maxTurns: 2,
  addons: repair(),
  instructions:
    "Pick one concrete, complicated task a person or team might face in a single day that would naturally benefit from being split across sub-agents running different models: multi-source research and synthesis, a multi-file coding task with distinct design/implementation/review phases, a structured report combining several independent analyses, or a multi-step data transformation pipeline. It must be self-contained (solvable from its description alone, no external files or live web access) and concrete enough to grade. Prefer variety across domains; choose an unusual domain. Provide 3-6 concrete, checkable success criteria.",
  output: s.object({
    title: s.string,
    domain: s.string,
    description: s.string,
    successCriteria: s.array(s.string),
  }),
});

// Agent role: single-call solver
const solveSingle = agent({
  name: "singleSolver",
  model: "medium",
  maxTurns: 1,
  instructions:
    "Solve the entire task yourself in one response, addressing every success criterion. No delegation, no tools. Return the full solution (use Markdown code fences where appropriate).",
  output: s.string,
});

const SKELETON = `import { agent, workflow, s } from "rig";

// Agent role: <role>
const analyze = agent({ model: "small", instructions: "...", output: s.string });

// Workflow role: combines the agents' outputs into the final solution
export default workflow({
  meta: { name: "decomposed-solution", description: "..." },
  body: async ({ call }) => {
    const analysis = await call(analyze, "...");
    return { solution: analysis ?? "" };
  },
});`;

const CONTRACT = `Rules for the program (you have no other documentation, follow exactly):
- Import only from "rig". Output only TypeScript source code, no prose; Markdown fences are tolerated.
- Use at least two agent() declarations, each preceded by a "// Agent role: ..." comment, with model "small" for simple sub-steps and "medium" or "large" for harder ones.
- Agents use \`output: s.string\` for any free-form text or source code. Do not wrap large solutions in object schemas.
- The root is export default of a workflow(). A workflow() spec accepts only meta, an optional input, and body; it has no output or execute field. The final { solution } object is simply what body returns. Take no required input. Do not invoke the root.
- body receives { call }; call(agent, inputString) returns the output or null on failure. Combine the agents' outputs into the final { solution: string }, rather than having one agent solve everything.
- Do not use console.log.
Required shape (abridged to one agent; yours needs at least two):

${SKELETON}`;

// Agent role: writes the decomposed rig program
const writer = agent({
  name: "decompositionWriter",
  model: "medium",
  maxTurns: 1,
  instructions: `You write a self-contained rig TypeScript program that solves a given task by splitting it across multiple agents.\n\n${CONTRACT}`,
  output: s.string,
});

// Agent role: fixes or fully generates the decomposed rig program
const fixer = agent({
  name: "decompositionFixer",
  model: "medium",
  maxTurns: 1,
  instructions: `You repair or generate rig TypeScript programs. When given source and an error, fix precisely what the error names while preserving what already worked; do not make unrelated edits. When told there is no source, generate the complete program from the task.\n\n${CONTRACT}`,
  output: s.string,
});

// Agent role: grader
const grader = agent({
  name: "grader",
  model: "large",
  maxTurns: 1,
  instructions:
    "Score two candidate solutions to the same task from 0-10 on how completely and correctly each satisfies the success criteria. Judge only correctness and completeness of content, not length and not which approach produced it. Pick winner 'single-call', 'decomposed', or 'tie' and explain the comparison.",
  output: s.object({
    singleScore: s.number,
    decomposedScore: s.number,
    winner: s.enum("single-call", "decomposed", "tie"),
    rationale: s.string,
  }),
});

type Attempt = {
  attempt: number;
  generation: string;
  typecheck: string;
  execute: string;
  output: string;
  note: string;
};

function stripFence(src: string): string {
  const m = src.match(/```(?:ts|typescript|rig)?\n([\s\S]*?)```/);
  return (m ? m[1] : src).trim();
}

export default workflow({
  meta: { name: "rig-decomposition-bench", description: "Single call vs decomposed rig program" },
  body: async ({ call }) => {
    const task = await call(pickTask, "Pick the benchmark task.");
    if (!task) return { status: "failed: task picker returned null" };
    const taskText = `Title: ${task.title}\nDomain: ${task.domain}\nDescription: ${task.description}\nSuccess criteria:\n${task.successCriteria.map((c: string) => `- ${c}`).join("\n")}`;

    const t0 = Date.now();
    const single = await call(solveSingle, taskText, { timeout: 5 * 60_000 });
    const singleMs = Date.now() - t0;
    const singleSolution = single ?? "";

    const t1 = Date.now();
    const attempts: Attempt[] = [];
    let source = "";
    let decomposedSolution = "";
    let passed = false;
    let lastError = "";
    for (let i = 1; i <= 2 && !passed; i++) {
      if (remaining() < 4 * 60_000) break;
      const rec: Attempt = { attempt: i, generation: "", typecheck: "skipped", execute: "skipped", output: "", note: "" };
      attempts.push(rec);
      let raw: string | null;
      if (i === 1) {
        raw = await call(writer, `Task:\n${taskText}`, { timeout: 5 * 60_000 });
      } else if (source.trim()) {
        raw = await call(
          fixer,
          `Task:\n${taskText}\n\nCurrent source:\n${source}\n\nCaptured error:\n${lastError}\n\nFix precisely what the error names, preserving what worked. Required skeleton:\n${SKELETON}`,
          { timeout: 5 * 60_000 },
        );
      } else {
        raw = await call(
          fixer,
          `There is no source to fix; the previous attempt produced nothing. Generate the complete program from scratch.\n\nTask:\n${taskText}\n\nRequired skeleton:\n${SKELETON}`,
          { timeout: 5 * 60_000 },
        );
      }
      const cleaned = raw && raw.trim() ? stripFence(raw) : "";
      if (!cleaned) {
        rec.generation = "empty (null or whitespace-only)";
        lastError = "";
        rec.note = i === 1 ? "Second attempt regenerates from scratch." : "";
        continue;
      }
      source = cleaned;
      rec.generation = `ok (${cleaned.length} chars)`;
      const tc = await runRig(cleaned, "--typecheck", 2 * 60_000);
      rec.typecheck = tc.status;
      if (!tc.ok) {
        lastError = truncate(tc.stdout + tc.stderr);
        rec.output = lastError;
        rec.note = "Next attempt fixes the typecheck error.";
        continue;
      }
      const ex = await runRig(cleaned, "--server", 5 * 60_000);
      rec.execute = ex.status;
      if (!ex.ok) {
        lastError = truncate(ex.stderr + (ex.stdout ? "\nstdout:\n" + ex.stdout : ""));
        rec.output = lastError;
        rec.note = "Next attempt fixes the execution error.";
        continue;
      }
      try {
        const parsed = JSON.parse(ex.stdout) as { solution?: unknown };
        if (typeof parsed.solution === "string" && parsed.solution.trim()) {
          decomposedSolution = parsed.solution;
          passed = true;
        } else {
          rec.execute = "failed";
          lastError = "stdout JSON had no non-empty string 'solution': " + truncate(ex.stdout);
          rec.output = lastError;
          rec.note = "Next attempt fixes the missing solution.";
        }
      } catch (e) {
        rec.execute = "failed";
        lastError = "stdout was not valid JSON: " + String(e) + "\n" + truncate(ex.stdout);
        rec.output = lastError;
        rec.note = "Next attempt fixes the output format.";
      }
    }
    const decomposedMs = Date.now() - t1;

    const decomposedForGrade = decomposedSolution || "(Decomposition produced no solution: no generated program executed successfully.)";
    const grade = await call(
      grader,
      `${taskText}\n\n=== SOLUTION A (single-call) ===\n${singleSolution || "(empty)"}\n\n=== SOLUTION B (decomposed) ===\n${decomposedForGrade}`,
      { timeout: 5 * 60_000 },
    );

    return {
      task,
      singleMs,
      decomposedMs,
      singleSolution,
      decomposedSolution,
      attempts,
      status: passed ? "pass" : "fail",
      decomposedSource: source,
      grade,
    };
  },
});