Negative Mean Kernel Duration

What is your question?

Hello,

I am trying to analyze some traces extracted by vLLM using the PyTorch profiler. When running TraceAnalysis on them (specifically get_gpu_kernel_breakdown()), I get some graphs that make sense, and some like the one in the image, which show a negative mean duration for certain kernels.

I have the code snippet I used for profiling pasted below. I am not sure where the issue is arising from. If this is a bug in HTA, it would be a good thing to look into.

Thanks for your attention.

Code

import logging
import os
import random
import time
from typing import Annotated
import json

import torch
from torch.profiler import profile, record_function, ProfilerActivity
import typer
from vllm import LLM, SamplingParams

# enable torch profiler, can also be set on cmd line
os.environ["VLLM_TORCH_PROFILER_RECORD_SHAPES"] = "1"

def get_sharegpt_prompts(path: str) -> list[str]:
    """Load prompts from ShareGPT dataset file."""
    return ... # removed for brevity

def main(
    model: Annotated[
        str, typer.Argument(help="Model name or path.")
    ] = "meta-llama/Llama-3.1-70B",
    dataset_path: Annotated[
        str, typer.Option(help="Path to dataset json")
    ] = "./datasets/ShareGPT_V3_unfiltered_cleaned_split.json",
    num_seqs: Annotated[int, typer.Option(help="Number of sequences to generate.")] = 32,
    max_num_batched_tokens: Annotated[
        int, typer.Option(help="Maximum number of batched tokens.")
    ] = 1024,
    max_num_seqs: Annotated[
        int, typer.Option(help="Maximum number of sequences.")
    ] = 256,
    max_length: Annotated[int, typer.Option(help="Maximum generation length.")] = 2048,
    num_warmup: Annotated[
        int, typer.Option(help="Number of warm-up iterations before measurement.")
    ] = 2,
    num_generations: Annotated[
        int, typer.Option(help="Number of generation iterations to run.")
    ] = 3,
    tensor_parallel_size: Annotated[
        int, typer.Option(help="Tensor parallel size for vLLM.")
    ] = 1,
    pipeline_parallel_size: Annotated[
        int, typer.Option(help="Pipeline parallel size for vLLM.")
    ] = 1,
    gpu_memory_utilization: Annotated[
        float, typer.Option(help="Target GPU memory utilization for vLLM.")
    ] = 0.7,
):
    if max_num_seqs > max_num_batched_tokens:
        logging.warning(
            "It is required to set --max-num-seqs <= --max-num-batched-tokens for the engine"
        )
        max_num_seqs = max_num_batched_tokens

    # Sample prompts.
    prompts = ... # removed for brevity

    # Create a sampling params object.
    sampling_params = SamplingParams(temperature=0.8, top_p=0.95)

    trace_dir = "./vllm_hta_traces"

    # Create an LLM.
    llm = LLM(
        model=model,
        tensor_parallel_size=tensor_parallel_size,
        max_model_len=max_length,
        # max_num_batched_tokens=max_num_batched_tokens,
        # max_num_seqs=max_num_seqs,
        gpu_memory_utilization=gpu_memory_utilization,
	profiler_config={
        	"profiler": "torch",
        	"torch_profiler_dir": trace_dir,
    }
    )

    # Warm up
    print("Warming up...")
    for _ in range(num_warmup):
        _ = llm.generate(prompts, sampling_params)

    outputs_ = []

    # Generate texts from the prompts.

    llm.start_profile()
    for i in range(num_generations):
        print(f"Profiling step {i}...")
        outputs_ = llm.generate(prompts, sampling_params)
    llm.stop_profile()


    # Add a buffer to wait for profiler in the background process
    # (in case MP is on) to finish writing profiling output.
    print("Waiting for trace files to flush...")
    time.sleep(10)


if __name__ == "__main__":
    app()

What have you tried?

No response

Environment

Ubuntu 24.02. Python 3.12.3. Installed HTA using pip.