Detect when CUDA Graph capture is ongoing at the time of kernel compilation or loading

Here's a way to trigger a CUDA error of not permitted operation during graph capturing. It would be great to detect that the CUDA graph capture is ongoing before we try to load/compile kernels and error out with a better error message.

import torch
from nvfuser import FusionDefinition, DataType

with FusionDefinition() as fd:
    T1 = fd.define_tensor(symbolic_sizes=[-1], contiguity=[True], dtype=DataType.Float, is_cpu=False)
    T2 = fd.define_tensor(symbolic_sizes=[-1], contiguity=[True], dtype=DataType.Float, is_cpu=False)
    T3 = fd.ops.add(T1, T2)
    fd.add_output(T3)

t1 = torch.randn(10, device="cuda")
t2 = torch.randn(10, device="cuda")

torch.cuda.synchronize()
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
stream.synchronize()
torch.cuda.current_stream().wait_stream(stream)
torch.cuda.synchronize()

# Skipping warmup going straight to CUDA graph capture
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
    out = fd.execute([t1, t2])

torch.cuda.synchronize()

error:

An error occurred while executing nvFuser FusionDefinition 0.
If you believe this is a bug or need assistance, please file an issue at https://github.com/NVIDIA/Fuser/issues/new
Here's a script to reproduce the error:

import torch
from nvfuser import FusionDefinition, DataType

def nvfuser_fusion_id0(fd : FusionDefinition) -> None :
    T0 = fd.define_tensor(symbolic_sizes=[-1], contiguity=[True], dtype=DataType.Float, is_cpu=False)
    T1 = fd.define_tensor(symbolic_sizes=[-1], contiguity=[True], dtype=DataType.Float, is_cpu=False)
    T2 = fd.ops.add(T0, T1)
    fd.add_output(T2)

with FusionDefinition() as fd:
    nvfuser_fusion_id0(fd)

inputs = [
    torch.randn((10,), dtype=torch.float32, device='cuda:0').as_strided((10,), (1,)),
    torch.randn((10,), dtype=torch.float32, device='cuda:0').as_strided((10,), (1,)),
]
fd.execute(inputs)

Traceback (most recent call last):
  File "/home/iyashchuk/dev/pytorch/master/nvfuser/__init__.py", line 111, in execute
    result = self._execute(inputs, override_user_schedule, device=device)
RuntimeError: _result == CUDA_SUCCESS INTERNAL ASSERT FAILED at "/home/iyashchuk/dev/nvfuser/csrc/executor_utils.cpp":1124, please report a bug to PyTorch. CUDA error: CUDA_ERROR_STREAM_CAPTURE_UNSUPPORTED failed with error operation not permitted when stream is capturing