model

genlm.eval.core.model

ModelResponse

Bases: BaseModel

Single model response containing generated text, probability, and optional metadata.

Source code in genlm/eval/core/model.py

class ModelResponse(BaseModel):
    """Single model response containing generated text, probability, and optional metadata."""

    response: Any
    weight: float
    metadata: Optional[Dict[str, Any]] = None

ModelOutput

Bases: BaseModel

Collection of model responses with execution metadata.

Source code in genlm/eval/core/model.py

class ModelOutput(BaseModel):
    """Collection of model responses with execution metadata."""

    responses: List[ModelResponse]
    runtime_seconds: Optional[float] = None
    metadata: Optional[Dict[str, Any]] = None

ModelAdaptor

Bases: Protocol

Protocol for async model adapters that process instances into model outputs.

Source code in genlm/eval/core/model.py

class ModelAdaptor(Protocol):
    """Protocol for async model adapters that process instances into model outputs."""

    async def __call__(
        self, instance: Instance, output_dir: str, replicate: int
    ) -> ModelOutput:
        """Process an instance and generate model outputs.

        Args:
            instance (Instance): Input dataset instance to process
            output_dir (str): Directory for saving any intermediate results
            replicate (int): Replicate index for multiple evaluation runs

        Returns:
            (ModelOutput): Model output containing responses and runtime information
        """
        ...  # pragma: no cover

__call__(instance, output_dir, replicate) async

Process an instance and generate model outputs.

Parameters:

Name	Type	Description	Default
instance	Instance	Input dataset instance to process	required
output_dir	str	Directory for saving any intermediate results	required
replicate	int	Replicate index for multiple evaluation runs	required

Returns:

Type	Description
ModelOutput	Model output containing responses and runtime information

Source code in genlm/eval/core/model.py

async def __call__(
    self, instance: Instance, output_dir: str, replicate: int
) -> ModelOutput:
    """Process an instance and generate model outputs.

    Args:
        instance (Instance): Input dataset instance to process
        output_dir (str): Directory for saving any intermediate results
        replicate (int): Replicate index for multiple evaluation runs

    Returns:
        (ModelOutput): Model output containing responses and runtime information
    """
    ...  # pragma: no cover