runtime¶

@abc.abstractmethod
def load(self) -> None:
    """Load model into memory and prepare for inference.

    This method should:
        - Load model weights from disk
        - Move model to target device
        - Perform warmup inference
        - Set model to evaluation mode

    Raises:
        FileNotFoundError: If model file does not exist.
        RuntimeError: If device is not available.
    """

@abc.abstractmethod
def infer(self, input: P) -> R:
    """Run inference on preprocessed input.

    Args:
        input: Preprocessed input ready for model inference.
            Type depends on backend (e.g., torch.Tensor for PyTorch).

    Returns:
        Raw model output. Type depends on model architecture.

    Raises:
        RuntimeError: If model is not loaded.

    Example:
        ```python
        with runtime:
            output = runtime.infer(input_tensor)
        ```
    """

@abc.abstractmethod
def unload(self) -> None:
    """Unload model and free resources.

    This method should:
        - Release model from memory
        - Clear device cache
        - Close any open handles

    Example:
        ```python
        runtime.load()
        # ... do inference ...
        runtime.unload()  # Free resources
        ```
    """

@contextlib.contextmanager
def context(self) -> t.Iterator[t.Self]:
    """Context manager for automatic lifecycle management.

    Automatically calls `load()` on entry and `unload()` on exit,
    even if an exception occurs.

    Yields:
        Self: The runtime instance.

    Example:
        ```python
        with runtime.context():
            result = runtime.infer(input)
        # Model is automatically unloaded here
        ```
    """
    self.load()
    try:
        yield self
    finally:
        self.unload()

def __enter__(self) -> t.Self:
    """Context manager entry.

    Loads the model.

    Returns:
        Self: The runtime instance.

    Example:
        ```python
        with runtime:  # Calls __enter__
            result = runtime.infer(input)
        ```
    """
    self.load()
    return self

def __exit__(
    self,
    exc_type: type[BaseException] | None,
    exc_val: BaseException | None,
    exc_tb: types.TracebackType | None,
) -> None:
    """Context manager exit.

    Unloads the model, even if an exception occurred.

    Args:
        exc_type: Exception type if an exception occurred.
        exc_val: Exception value if an exception occurred.
        exc_tb: Exception traceback if an exception occurred.
    """
    self.unload()

@abc.abstractmethod
def infer_batch(self, inputs: list[P]) -> list[R]:
    """Run inference on a batch of inputs.

    Process multiple inputs in a single forward pass for better
    throughput. Inputs should already have batch dimension.

    Args:
        inputs: List of preprocessed inputs. Each input should have
            shape (1, ...) for proper batching.

    Returns:
        List of raw outputs, one per input. Each output maintains
        the batch dimension (1, ...).

    Raises:
        RuntimeError: If model is not loaded.

    Example:
        ```python
        with runtime:
            # Prepare batch
            batch = [
                torch.randn(1, 3, 224, 224),
                torch.randn(1, 3, 224, 224),
                torch.randn(1, 3, 224, 224),
            ]

            # Batch inference
            results = runtime.infer_batch(batch)

            # results[0], results[1], results[2] correspond to inputs
        ```
    """

def infer(self, input: P) -> R:
    """Single inference (delegates to batch inference).

    Wraps the input in a list, calls `infer_batch()`, and returns
    the first result. This provides a convenient single-input API
    while reusing the batch implementation.

    Args:
        input: Preprocessed input ready for model inference.

    Returns:
        Raw model output.

    Raises:
        RuntimeError: If model is not loaded.

    Example:
        ```python
        with runtime:
            # These are equivalent:
            result = runtime.infer(input)
            result = runtime.infer_batch([input])[0]
        ```
    """
    results = self.infer_batch([input])
    return results[0]