[Bug]: On Windows, parallel runs that share the transform cache fail to load a test file with EPERM

// Starts four `playwright test --list` shards at once over one cold transform
// cache, ROUNDS times, then the same with a cache folder per shard, and
// counts the rounds in which a shard failed to load its tests.
//
//   node race.mjs [rounds] [project dir]
import { spawn } from 'node:child_process';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';

const rounds = Number(process.argv[2] ?? 100);
const project = path.resolve(process.argv[3] ?? '.');
const cli = path.join(project, 'node_modules', '@playwright', 'test', 'cli.js');

// Thirty small specs, written once.
const tests = path.join(project, 'tests');
fs.mkdirSync(tests, { recursive: true });
for (let i = 1; i <= 30; i++) {
  const file = path.join(tests, `spec${String(i).padStart(2, '0')}.spec.ts`);
  if (!fs.existsSync(file)) {
    fs.writeFileSync(
      file,
      `import { test, expect } from '@playwright/test';\n\n` +
        `test('spec ${i}', async () => {\n  expect(${i}).toBe(${i});\n});\n`,
    );
  }
}

function shard(index, cacheDir) {
  return new Promise((resolve) => {
    const child = spawn(process.execPath, [cli, 'test', '--list', `--shard=${index}/4`], {
      cwd: project,
      env: { ...process.env, PWTEST_CACHE_DIR: cacheDir },
    });
    let output = '';
    child.stdout.on('data', (chunk) => (output += chunk));
    child.stderr.on('data', (chunk) => (output += chunk));
    child.on('close', (code) => resolve({ index, code, output }));
  });
}

const failedRounds = { shared: 0, separate: 0 };
for (let round = 1; round <= rounds; round++) {
  for (const mode of ['shared', 'separate']) {
    // A fresh folder each round, so every shard starts with nothing compiled,
    // as on a fresh checkout.
    const base = fs.mkdtempSync(path.join(os.tmpdir(), 'pw-cache-race-'));
    const results = await Promise.all(
      [1, 2, 3, 4].map((i) => shard(i, mode === 'shared' ? base : path.join(base, `shard-${i}`))),
    );
    const failed = results.filter((r) => r.code !== 0);
    if (failed.length > 0) failedRounds[mode]++;
    for (const r of failed) {
      const lines = r.output.split('\n').filter((line) => /Error|^\s+at /.test(line));
      console.log(`round ${round}, ${mode} cache, shard ${r.index}/4 exited ${r.code}:\n${lines.slice(0, 4).join('\n')}`);
    }
    fs.rmSync(base, { recursive: true, force: true });
  }
}
console.log(`one cache folder shared by the four shards: ${failedRounds.shared} of ${rounds} rounds had a shard fail`);
console.log(`a cache folder per shard: ${failedRounds.separate} of ${rounds} rounds had a shard fail`);

node hammer-traced.cjs <path to node_modules/playwright> <cache dir> 5000 <start time> <trace file> in four processes at once, with one cache dir and one start time a few seconds ahead, then node line-up.mjs trace-*.json:

write of .map by pid 40136: EPERM after 270 us; meanwhile on that file: pid 40660 deleting, pid 29512 deleting, pid 31032 deleting
write of .js by pid 40136: EPERM after 710 us; meanwhile on that file: pid 40660 deleting, pid 31032 writing, pid 31032 deleting
write of .js by pid 40136: EPERM after 289 us; meanwhile on that file: pid 29512 deleting
write of .js by pid 40660: EPERM after 175 us; meanwhile on that file: pid 29512 deleting
...
17984 deletes and writes traced across 4 processes. 9 writes failed, 9 of them while another process was deleting that same file. (118 deletes failed too; Playwright ignores those.)

The same loop untraced, with no second process: 16297 calls in 20 s, none failed. Four processes with a cache dir each: 8513 calls in 5 s, none failed.

hammer-traced.cjs

// Calls Playwright's own addToCache for one file over and over, recording when
// each delete and write of the cached files began and ended, on the machine's
// one clock, so a failed write can be lined up against what the other
// processes were doing to that file at that moment.
//
//   node hammer-traced.cjs <playwright package dir> <cache dir> <milliseconds> <start at, epoch ms> <trace file>
const fs = require('node:fs');
const path = require('node:path');
const crypto = require('node:crypto');

const [playwrightDir, cacheDir, duration, startAt, traceFile] = process.argv.slice(2);
process.env.PWTEST_CACHE_DIR = cacheDir;

const writeFileSync = fs.writeFileSync;
const events = [];
for (const op of ['rmSync', 'writeFileSync']) {
  const original = fs[op];
  fs[op] = function (file, ...rest) {
    const start = process.hrtime.bigint();
    let result = 'ok';
    try {
      return original.call(this, file, ...rest);
    } catch (error) {
      result = error.code;
      throw error;
    } finally {
      events.push([op, String(file), String(start), String(process.hrtime.bigint()), result]);
    }
  };
}

const { cc } = require(path.join(playwrightDir, 'lib', 'common', 'index.js'));
const code = 'x'.repeat(40_000);
const map = { version: 3, mappings: 'A'.repeat(40_000) };
const contentHash = crypto.createHash('sha1').update('content').digest('hex');
const { addToCache } = cc.getFromCompilationCache('C:\\nowhere\\journey.spec.ts', contentHash, undefined);
while (Date.now() < Number(startAt));
const end = Date.now() + Number(duration);
while (Date.now() < end) {
  try {
    addToCache(code, map);
  } catch {}
}
writeFileSync(traceFile, JSON.stringify({ pid: process.pid, events }));

line-up.mjs

// For each write that failed in the traces (the one failure Playwright does
// not swallow), what the other processes were doing to that same file while
// it ran.
//
//   node line-up.mjs <trace file>...
import fs from 'node:fs';

const traces = process.argv.slice(2).map((f) => JSON.parse(fs.readFileSync(f, 'utf8')));
const all = traces.flatMap((t) =>
  t.events.map(([op, file, start, end, result]) => ({ pid: t.pid, op, file, start: BigInt(start), end: BigInt(end), result })),
);
const failedWrites = all.filter((e) => e.op === 'writeFileSync' && e.result !== 'ok');
const failedDeletes = all.filter((e) => e.op === 'rmSync' && e.result !== 'ok');
let duringDelete = 0;
for (const f of failedWrites) {
  const overlapping = all.filter(
    (e) => e.pid !== f.pid && e.file === f.file && e.start <= f.end && e.end >= f.start,
  );
  if (overlapping.some((e) => e.op === 'rmSync')) duringDelete++;
  const ext = f.file.slice(f.file.lastIndexOf('.'));
  console.log(
    `write of ${ext} by pid ${f.pid}: ${f.result} after ${(f.end - f.start) / 1000n} us; meanwhile on that file: ` +
      (overlapping.map((e) => `pid ${e.pid} ${e.op === 'rmSync' ? 'deleting' : 'writing'}`).join(', ') ||
        'nothing from another process'),
  );
}
console.log(
  `${all.length} deletes and writes traced across ${traces.length} processes. ` +
    `${failedWrites.length} writes failed, ${duringDelete} of them while another process was deleting that same file. ` +
    `(${failedDeletes.length} deletes failed too; Playwright ignores those.)`,
);