The first command of a process never completes when processes start on several GPUs at once (Arc Pro B70, xe)

Start with the Level Zero reproducer in the issue and the reported stack in IoctlHelperXe::xeWaitUserFence, then trace how the first immediate command is submitted on the xe driver. Compare runs with direct submission enabled and disabled, using the reported multi-GPU process setup to reproduce the hang. Done means identifying why the fill is not executed and verifying a fix against the reproducer.

Summary

When fresh processes start at the same time, one per GPU, the first command of a process sometimes never completes. That command is a memory fill on an immediate command list, waited for with zeEventHostSynchronize. The host waits forever in xeWaitUserFence, and the fence still holds its initial value: the GPU never ran the fill.

Meanwhile the whole GPU stops executing new work. A new process on the same GPU also hangs on its first command, while the other GPUs work normally. The GPU recovers as soon as the stuck process exits.

With one GPU and processes started one after another, it does not happen.

A Level Zero program of about 60 lines reproduces it in about 1% of processes. It is not specific to one runtime: we first saw it in oneCCL's communicator initialization, then reproduced it with SYCL and with Level Zero directly.

Platform

• 4× Intel Arc Pro B70 (PCI 8086:e223), plus an ASPEED BMC display.
• Linux 6.17.0-1009-intel, xe kernel driver (srcversion E3C9808155FEF5AE5721170).
• Compute runtime (NEO) 26.40.40196.8 (libze_intel_gpu.so.1.18.40196), Level Zero loader 1.34.0. Also reproduced with NEO 26.35.39758.10.

Reproducer

l0repro.c (below) forks one fresh process per GPU, all at once. Each process does zeInit, then on its GPU:

1. creates a context, an immediate command list (ordinal 0, asynchronous) and an 8 KiB device allocation;
2. creates a host-visible event;
3. appends zeCommandListAppendMemoryFill signalling the event;
4. waits with zeEventHostSynchronize(ev, UINT64_MAX);
5. exits.

A process still waiting after 30 s (alarm) is counted as hung. Build and run:

gcc -O2 -I<level_zero include dir> l0repro.c -o l0repro -lze_loader
./l0repro 100 4   # 100 iterations x 4 GPUs = 400 processes
./l0repro 400 1   # 400 processes, one GPU, one after another

// The first command of a fresh process on Level Zero directly: an immediate
// command list, a memory fill, and a host wait on its event. One fresh process
// per GPU, at once; a child still waiting after 30 s is counted as hung.
#include <level_zero/ze_api.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/wait.h>
#include <unistd.h>

#define OK(x) do { ze_result_t r_ = (x); if (r_) { fprintf(stderr, "%s -> 0x%x\n", #x, r_); return 3; } } while (0)

static int child(int dev) {
    OK(zeInit(ZE_INIT_FLAG_GPU_ONLY));
    uint32_t n = 1; ze_driver_handle_t drv; OK(zeDriverGet(&n, &drv));
    uint32_t nd = 0; OK(zeDeviceGet(drv, &nd, NULL));
    if (nd <= (uint32_t)dev) return 2;
    ze_device_handle_t devs[16]; OK(zeDeviceGet(drv, &nd, devs));
    ze_context_desc_t cd = {ZE_STRUCTURE_TYPE_CONTEXT_DESC};
    ze_context_handle_t ctx; OK(zeContextCreate(drv, &cd, &ctx));
    ze_command_queue_desc_t qd = {ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC};
    qd.ordinal = 0; qd.mode = ZE_COMMAND_QUEUE_MODE_ASYNCHRONOUS;
    ze_command_list_handle_t cl; OK(zeCommandListCreateImmediate(ctx, devs[dev], &qd, &cl));
    ze_device_mem_alloc_desc_t md = {ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC};
    void *p; OK(zeMemAllocDevice(ctx, &md, 8192, 64, devs[dev], &p));
    ze_event_pool_desc_t pd = {ZE_STRUCTURE_TYPE_EVENT_POOL_DESC}; pd.count = 1; pd.flags = ZE_EVENT_POOL_FLAG_HOST_VISIBLE;
    ze_event_pool_handle_t pool; OK(zeEventPoolCreate(ctx, &pd, 1, &devs[dev], &pool));
    ze_event_desc_t ed = {ZE_STRUCTURE_TYPE_EVENT_DESC}; ed.signal = ZE_EVENT_SCOPE_FLAG_HOST;
    ze_event_handle_t ev; OK(zeEventCreate(pool, &ed, &ev));
    int zero = 0;
    OK(zeCommandListAppendMemoryFill(cl, p, &zero, 1, 8192, ev, 0, NULL));
    OK(zeEventHostSynchronize(ev, UINT64_MAX));
    return 0;
}

int main(int argc, char **argv) {
    int iters = argc > 1 ? atoi(argv[1]) : 100, ndev = argc > 2 ? atoi(argv[2]) : 4, hung = 0, failed = 0;
    for (int it = 0; it < iters; it++) {
        pid_t pids[16];
        for (int d = 0; d < ndev; d++) if ((pids[d] = fork()) == 0) { alarm(30); _exit(child(d)); }
        for (int d = 0; d < ndev; d++) {
            int st = 0; waitpid(pids[d], &st, 0);
            if (WIFSIGNALED(st) && WTERMSIG(st) == SIGALRM) { hung++; printf("iter %d dev %d: HUNG\n", it, d); fflush(stdout); }
            else if (!WIFEXITED(st) || WEXITSTATUS(st)) { failed++; printf("iter %d dev %d: status 0x%x\n", it, d, st); }
        }
    }
    printf("DONE %d iterations, %d hung, %d failed\n", iters, hung, failed);
    return hung || failed;
}

Results (hung processes)

Hangs come in streaks: once a GPU is stuck, the next fresh processes on that GPU hang too, until the stuck one has exited. Disabling direct submission makes the hang rarer, but it still happens.

What the stuck process shows

The main thread is in the event host wait:

#0  ioctl
#1  NEO::SysCalls::ioctl                          sys_calls_linux.cpp:110
#2  NEO::Drm::ioctl                               drm_neo.cpp:132
#3  NEO::IoctlHelperXe::xeWaitUserFence           ioctl_helper_xe.cpp:848
#4  NEO::IoctlHelperXe::waitUserFence             ioctl_helper_xe.cpp:860
#5  NEO::Drm::waitUserFence                       drm_neo.cpp:1204
#6  NEO::Drm::waitOnUserFencesImpl
#7  NEO::Drm::waitOnUserFences                    drm_neo.cpp:2051
#8  NEO::DrmCommandStreamReceiver<Xe2HpgCoreFamily>::waitUserFence  drm_command_stream.inl:371
#9  L0::EventImp<unsigned long>::tryUserFenceWaitForHostSynchronize event_impl.inl:485
#10 L0::EventImp<unsigned long>::hostSynchronize  event_impl.inl:1073

The values of xeWaitUserFence's arguments, read with gdb in frame #3:

• ctxId = 1, op = 3, value = 1;
• userInterrupt = false, externalInterruptId = 0xffffffff.

The fence at addr reads 0, so the wait is not a missed wake-up: the work never executed.

The process's only other driver thread is the DirectSubmissionController thread, in its usual wait or sleep.

Impact

Every multi-process, multi-GPU job starts this way: one rank per GPU, all at once, each with a first small command at initialization. With oneCCL, a 2-rank creation hangs about once in 1800. The rank that hangs blocks its GPU, and its peers wait for it.

BTW, here is l0repro.c:

// The first command of a fresh process on Level Zero directly: an immediate
// command list, a memory fill, and a host wait on its event. One fresh process
// per GPU, at once; a child still waiting after 30 s is counted as hung.
#include <level_zero/ze_api.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/wait.h>
#include <unistd.h>

#define OK(x) do { ze_result_t r_ = (x); if (r_) { fprintf(stderr, "%s -> 0x%x\n", #x, r_); return 3; } } while (0)

static int child(int dev) {
    OK(zeInit(ZE_INIT_FLAG_GPU_ONLY));
    uint32_t n = 1; ze_driver_handle_t drv; OK(zeDriverGet(&n, &drv));
    uint32_t nd = 0; OK(zeDeviceGet(drv, &nd, NULL));
    if (nd <= (uint32_t)dev) return 2;
    ze_device_handle_t devs[16]; OK(zeDeviceGet(drv, &nd, devs));
    ze_context_desc_t cd = {ZE_STRUCTURE_TYPE_CONTEXT_DESC};
    ze_context_handle_t ctx; OK(zeContextCreate(drv, &cd, &ctx));
    ze_command_queue_desc_t qd = {ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC};
    qd.ordinal = 0; qd.mode = ZE_COMMAND_QUEUE_MODE_ASYNCHRONOUS;
    ze_command_list_handle_t cl; OK(zeCommandListCreateImmediate(ctx, devs[dev], &qd, &cl));
    ze_device_mem_alloc_desc_t md = {ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC};
    void *p; OK(zeMemAllocDevice(ctx, &md, 8192, 64, devs[dev], &p));
    ze_event_pool_desc_t pd = {ZE_STRUCTURE_TYPE_EVENT_POOL_DESC}; pd.count = 1; pd.flags = ZE_EVENT_POOL_FLAG_HOST_VISIBLE;
    ze_event_pool_handle_t pool; OK(zeEventPoolCreate(ctx, &pd, 1, &devs[dev], &pool));
    ze_event_desc_t ed = {ZE_STRUCTURE_TYPE_EVENT_DESC}; ed.signal = ZE_EVENT_SCOPE_FLAG_HOST;
    ze_event_handle_t ev; OK(zeEventCreate(pool, &ed, &ev));
    int zero = 0;
    OK(zeCommandListAppendMemoryFill(cl, p, &zero, 1, 8192, ev, 0, NULL));
    OK(zeEventHostSynchronize(ev, UINT64_MAX));
    return 0;
}

int main(int argc, char **argv) {
    int iters = argc > 1 ? atoi(argv[1]) : 100, ndev = argc > 2 ? atoi(argv[2]) : 4, hung = 0, failed = 0;
    for (int it = 0; it < iters; it++) {
        pid_t pids[16];
        for (int d = 0; d < ndev; d++) if ((pids[d] = fork()) == 0) { alarm(30); _exit(child(d)); }
        for (int d = 0; d < ndev; d++) {
            int st = 0; waitpid(pids[d], &st, 0);
            if (WIFSIGNALED(st) && WTERMSIG(st) == SIGALRM) { hung++; printf("iter %d dev %d: HUNG\n", it, d); fflush(stdout); }
            else if (!WIFEXITED(st) || WEXITSTATUS(st)) { failed++; printf("iter %d dev %d: status 0x%x\n", it, d, st); }
        }
    }
    printf("DONE %d iterations, %d hung, %d failed\n", iters, hung, failed);
    return hung || failed;
}