[rig-decomposition-bench] Daily rig decomposition benchmark — 2026-10-27 — tie

Error: spawn node ENOENT

Error: spawn node ENOENT

The solver returned an empty string (no content). Nothing to display.

Empty — execution never ran (typecheck subprocess failed with spawn node ENOENT on both attempts).

import { agent, workflow, s, repair, configureAgent, copilotEngine } from "rig";

configureAgent(copilotEngine({ server: true }));

const DEADLINE = Date.now() + 25 * 60_000;
const remaining = () => DEADLINE - Date.now();

const SKELETON = `import { agent, workflow, s } from "rig";

// Agent role: <role>
const analyze = agent({ model: "small", instructions: "...", output: s.string });

// Workflow role: combines the agents' outputs into the final solution
export default workflow({
  meta: { name: "decomposed-solution", description: "..." },
  body: async ({ call }) => {
    const analysis = await call(analyze, "...");
    return { solution: analysis ?? "" };
  },
});`;

const CONTRACT = `Write ONE self-contained TypeScript program for the "rig" library. Rules:
- Import ONLY from "rig": import { agent, workflow, s } from "rig";
- Define at least TWO agents, each preceded by a comment "// Agent role: ...". Use model "small" for simple sub-steps and "medium" or "large" for harder ones. Every agent uses output: s.string (free-form text / source), never an object wrapper for large content.
- The root must be \`export default workflow({...})\`, exported without invoking it, taking NO required input.
- A workflow() spec accepts ONLY meta ({name, description}), an optional input, and body. It has NO output or execute field. The final { solution: string } object is simply what body returns.
- body is async ({ call }) => {...}; use \`await call(agentFn, "prompt string")\` which returns string or null; coalesce null with ?? "".
- The final answer must COMBINE outputs of multiple agents (not one agent solving everything).
- Alternatively an agent() root must declare output: s.object({ solution: s.string }), but prefer the workflow.
- Output only the program source, no commentary. Markdown fences are tolerated.
Required shape (abridged to one agent; yours needs at least two):
${SKELETON}`;

// Agent role: pick a concrete benchmark task with success criteria.
const picker = agent({
  name: "picker",
  model: "small",
  maxTurns: 2,
  addons: [repair()],
  instructions:
    "Pick ONE concrete, complicated task a person or team might face in a single day that would naturally benefit from being split across sub-agents running different models (multi-source research and synthesis, multi-file coding with design/implementation/review, structured multi-analysis report, or multi-step data transformation). It must be self-contained (no external files or live web) and gradable. Vary the domain; choose something unusual. Return title, domain, description, and 3-6 concrete checkable success criteria.",
  output: s.object({
    title: s.string,
    domain: s.string,
    description: s.string,
    criteria: s.array(s.string),
  }),
});

// Agent role: solve the whole task alone in a single turn.
const solver = agent({
  name: "solver",
  model: "medium",
  maxTurns: 1,
  instructions:
    "Solve the entire task yourself in one response, explicitly addressing every success criterion. No delegation, no tools. Return the complete solution.",
  output: s.string,
});

// Agent role: write (or fix) a decomposed rig program.
const writer = agent({
  name: "writer",
  model: "medium",
  maxTurns: 1,
  instructions: CONTRACT,
  output: s.string,
});

// Agent role: grade both solutions.
const grader = agent({
  name: "grader",
  model: "large",
  maxTurns: 1,
  instructions:
    "Score each solution 0-10 on how completely and correctly it satisfies the success criteria. Judge only correctness and completeness of content — not length, and not which approach produced it. Pick winner 'single-call', 'decomposed', or 'tie' and explain.",
  output: s.object({
    singleScore: s.number,
    decomposedScore: s.number,
    winner: s.enum("single-call", "decomposed", "tie"),
    rationale: s.string,
  }),
});

type Attempt = {
  attempt: number;
  generation: string;
  typecheck: string;
  execute: string;
  error: string;
};

function stripFences(src: string): string {
  const m = src.match(/```(?:ts|typescript|rig)?\s*\n([\s\S]*?)```/);
  return (m ? m[1]! : src).trim();
}

async function runRig(source: string, flag: string, limitMs: number) {
  const { spawn } = await import("node:child_process");
  return new Promise<{ code: number | null; stdout: string; stderr: string; timedOut: boolean }>((resolve) => {
    const child = spawn("node", ["rig.ts", flag], { cwd: ".github/skills/rig" });
    let stdout = "";
    let stderr = "";
    let timedOut = false;
    const t = setTimeout(() => {
      timedOut = true;
      child.kill("SIGKILL");
    }, Math.max(1000, limitMs));
    child.stdout.on("data", (d: unknown) => { stdout += String(d); });
    child.stderr.on("data", (d: unknown) => { stderr += String(d); });
    child.on("error", (e: Error) => { clearTimeout(t); resolve({ code: -1, stdout, stderr: stderr + String(e), timedOut }); });
    child.on("close", (code: number | null) => { clearTimeout(t); resolve({ code, stdout, stderr, timedOut }); });
    child.stdin.on("error", () => {});
    child.stdin.end(source);
  });
}

// Workflow role: orchestrate pick, single-call, decomposed, and grading phases deterministically.
export default workflow({
  meta: {
    name: "decomposition-benchmark",
    description: "Compare single-call vs decomposed solutions of a picked task",
    phases: ["Pick", "Single", "Decompose", "Grade"],
  },
  body: async ({ call, phase }) => {
    phase("Pick");
    const task = await call(picker, "Pick the task now.");
    if (!task) {
      return { task: null, error: "task picker returned null", status: "fail", attempts: [] as Attempt[] };
    }
    const taskText = `Title: ${task.title}\nDomain: ${task.domain}\nDescription: ${task.description}\nSuccess criteria:\n${task.criteria.map((c: string, i: number) => `${i + 1}. ${c}`).join("\n")}`;

    phase("Single");
    const t0 = Date.now();
    const single = (await call(solver, taskText)) ?? "";
    const singleMs = Date.now() - t0;

    phase("Decompose");
    const d0 = Date.now();
    const attempts: Attempt[] = [];
    let source = "";
    let solution = "";
    let status = "fail";
    let lastError = "";
    for (let n = 1; n <= 2; n++) {
      if (remaining() < 3 * 60_000) {
        attempts.push({ attempt: n, generation: "skipped (time budget)", typecheck: "skipped", execute: "skipped", error: "" });
        break;
      }
      let prompt: string;
      if (n === 1) {
        prompt = `Task:\n${taskText}\n\nWrite the decomposed rig program.`;
      } else if (source.trim()) {
        prompt = `Task:\n${taskText}\n\nFix this program. Fix precisely what the error names and preserve what already worked.\n\nRequired shape:\n${SKELETON}\n\nProgram:\n${source}\n\nError:\n${lastError}`;
      } else {
        prompt = `Task:\n${taskText}\n\nThere is no source to fix; the previous attempt produced nothing. Generate the complete program from the task and this required shape:\n${SKELETON}`;
      }
      const raw = await call(writer, prompt);
      const cleaned = raw ? stripFences(raw) : "";
      if (!cleaned.trim()) {
        attempts.push({ attempt: n, generation: "empty", typecheck: "skipped", execute: "skipped", error: "writer returned null/empty" });
        lastError = "";
        continue;
      }
      source = cleaned;
      const rec: Attempt = { attempt: n, generation: "ok", typecheck: "skipped", execute: "skipped", error: "" };
      attempts.push(rec);
      const tc = await runRig(source, "--typecheck", Math.min(2 * 60_000, remaining()));
      if (tc.code !== 0) {
        rec.typecheck = tc.timedOut ? "timeout" : "fail";
        rec.error = (tc.stdout + tc.stderr).trim() || "typecheck failed";
        lastError = rec.error;
        continue;
      }
      rec.typecheck = "pass";
      if (remaining() < 60_000) { rec.execute = "skipped (time budget)"; break; }
      const ex = await runRig(source, "--server", Math.min(5 * 60_000, remaining()));
      if (ex.code !== 0) {
        rec.execute = ex.timedOut ? "timeout" : "fail";
        rec.error = (ex.stdout + "\n" + ex.stderr).trim();
        lastError = rec.error;
        continue;
      }
      rec.execute = "pass";
      try {
        const parsed = JSON.parse(ex.stdout.trim());
        solution = typeof parsed?.solution === "string" ? parsed.solution : JSON.stringify(parsed);
      } catch {
        solution = ex.stdout.trim();
      }
      status = solution.trim() ? "pass" : "fail";
      break;
    }
    const decomposedMs = Date.now() - d0;

    phase("Grade");
    const decomposedForGrade = solution.trim() ? solution : "(Decomposition produced no solution.)";
    const grade = remaining() > 0
      ? await call(grader, `${taskText}\n\n=== SOLUTION A (single-call) ===\n${single || "(empty)"}\n\n=== SOLUTION B (decomposed) ===\n${decomposedForGrade}`)
      : null;

    return { task, singleMs, decomposedMs, single, decomposedSolution: solution, attempts, status, grade, decomposedSource: source };
  },
});