[GENERAL SUPPORT]: BilogY requires observations - error when running BoToch model with client.run_trials()

Run the provided minimal reproduction first, then inspect ax/api/client.py and ax/service/scheduler.py from the traceback. Follow how the experiment data reaches the TorchAdapter and why it becomes None despite experiment.to_df() containing data. Done means client.run_trials() completes the requested trials without SchedulerInternalError or the BilogY observations exception.

Question

Dear Ax team,

I am trying to figure out why running my generation strategy through the Scheduler class (with def client.run_trials()) results in the following error message:
Traceback (most recent call last): File "/home/ruard/dunia-ml/minimal_ax_test.py", line 154, in <module> main() ^^^^^^ File "/home/ruard/dunia-ml/minimal_ax_test.py", line 144, in main client.run_trials( File "/home/ruard/dunia-ml/.venv/lib/python3.11/site-packages/ax/api/client.py", line 652, in run_trials scheduler.run_n_trials(max_trials=max_trials) File "/home/ruard/dunia-ml/.venv/lib/python3.11/site-packages/ax/service/scheduler.py", line 593, in run_n_trials for _ in self.run_trials_and_yield_results( File "/home/ruard/dunia-ml/.venv/lib/python3.11/site-packages/ax/service/scheduler.py", line 720, in run_trials_and_yield_results while self._num_remaining_requested_trials > 0 and self.run( ^^^^^^^^^ File "/home/ruard/dunia-ml/.venv/lib/python3.11/site-packages/ax/service/scheduler.py", line 1132, in run raise SchedulerInternalError( ax.service.scheduler.SchedulerInternalError: No trials are running but model requires more data. This is an invalid state of the scheduler, as no more trials can be produced but also no more data is expected as there are no running trials. This should be investigated.

I have debugged it so far to figure out that basically the Experiment object does have actual data inside (checked through experiment.to_df()), however the data object passed down somewhere in the code is None and this results the final observations passed to the TorchAdapter to be an empty list (therefore resulting in the DataException 'BilogY requires observations.').

Can you help me fix this problem?

Please provide any relevant code snippet if applicable.

from ax import Client, ChoiceParameterConfig, RangeParameterConfig
from ax.generation_strategy.center_generation_node import CenterGenerationNode
from ax.generation_strategy.generation_node import GenerationNode
from ax.generation_strategy.generation_strategy import GenerationStrategy
from ax.generation_strategy.model_spec import GeneratorSpec
from ax.generation_strategy.transition_criterion import MinTrials
from ax.modelbridge.registry import Generators
from ax.models.torch.botorch_modular.surrogate import ModelConfig, SurrogateSpec
from ax.api.protocols.runner import IRunner, TrialStatus
from ax.api.types import TParameterization
from botorch.acquisition.logei import qLogExpectedImprovement
from botorch.models import SingleTaskGP
from typing import Any, Dict
import uuid


# Simple objective function
def objective_function(x1: float, x2: float, category: str) -> float:
    """Simple test function that depends on parameters."""
    if category == "A":
        return x1**2 + x2**2
    elif category == "B":
        return x1**2 - x2**2
    else:
        raise ValueError(f"Invalid category: {category}")


# Simple runner that evaluates the objective function
class SimpleRunner(IRunner):
    def run_trial(
        self, trial_index: int, parameterization: TParameterization
    ) -> Dict[str, Any]:
        """Run a trial by evaluating the objective function."""
        result = objective_function(
            parameterization["x1"], 
            parameterization["x2"], 
            parameterization["category"]
        )
        return {"objective": result}

    def poll_trial(
        self, trial_index: int, trial_metadata: Dict[str, Any]
    ) -> TrialStatus:
        """Trials complete immediately."""
        return TrialStatus.COMPLETED

    def stop_trial(
        self, trial_index: int, trial_metadata: Dict[str, Any]
    ) -> Dict[str, Any]:
        """Stop a trial."""
        return trial_metadata


def create_generation_strategy():
    """Create a generation strategy with Center -> Sobol -> BoTorch phases."""
    
    # Create BoTorch node with SingleTaskGP
    bo_node = GenerationNode(
        node_name="BOTORCH_MODULAR",
        model_specs=[
            GeneratorSpec(
                model_enum=Generators.BOTORCH_MODULAR,
                model_kwargs={
                    "surrogate_spec": SurrogateSpec(
                        model_configs=[
                            ModelConfig(
                                botorch_model_class=SingleTaskGP,
                                model_options={},
                                input_transform_classes=[],
                                outcome_transform_classes=[],
                            )
                        ]
                    ),
                    "refit_on_cv": True,
                    "warm_start_refit": False,
                    "botorch_acqf_class": qLogExpectedImprovement,
                },
                model_gen_kwargs={
                    "model_gen_options": {
                        "optimizer_kwargs": {
                            "options": {"with_grad": True}
                        },
                    },
                },
            ),
        ],
    )
    
    # Create Sobol node with transition to BoTorch
    sobol_node = GenerationNode(
        node_name="Sobol",
        model_specs=[
            GeneratorSpec(
                model_enum=Generators.SOBOL,
                model_kwargs={"seed": 42},
            ),
        ],
        transition_criteria=[
            MinTrials(
                threshold=5,  # Transition after 5 Sobol trials
                transition_to=bo_node.node_name,
                use_all_trials_in_exp=True,
            )
        ],
    )
    
    # Create center node
    center_node = CenterGenerationNode(next_node_name=sobol_node.node_name)
    
    # Create generation strategy
    generation_strategy = GenerationStrategy(
        name="Center_Sobol_BoTorch",
        nodes=[center_node, sobol_node, bo_node],
    )
    
    return generation_strategy


def main():
    """Main function to demonstrate the issue."""
    
    # Create experiment parameters
    parameters = [
        RangeParameterConfig(name="x1", parameter_type="float", bounds=(0, 1)),
        RangeParameterConfig(name="x2", parameter_type="float", bounds=(0, 1)),
        ChoiceParameterConfig(name="category", parameter_type="str", values=["A", "B"]),
    ]
    
    # Create client and configure experiment
    client = Client()
    experiment_name = f"test-experiment-{uuid.uuid4()}"
    
    client.configure_experiment(
        name=experiment_name,
        parameters=parameters,
    )
    client.configure_optimization(objective="objective")
    client.set_generation_strategy(create_generation_strategy())
    client.configure_runner(SimpleRunner())
    
    print(f"Created experiment: {experiment_name}")
    print("Running trials...")
    
    client.run_trials(
        max_trials=10,  # More than the 5 Sobol trials + 1 center trial
        parallelism=1,
        tolerated_trial_failure_rate=0.1,
        initial_seconds_between_polls=1,
    )



if __name__ == "__main__":
    main()

Code of Conduct

• I agree to follow this Ax's Code of Conduct