Skip to content

evaluator

genlm.eval.domains.json_schema.evaluator

JSONSchemaBenchEvaluator

Bases: Evaluator[JSONSchemaBenchInstance]

Evaluator for JSON schema.

Source code in genlm/eval/domains/json_schema/evaluator.py 
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
class JSONSchemaBenchEvaluator(Evaluator[JSONSchemaBenchInstance]):
    """Evaluator for JSON schema."""

    def evaluate_sample(self, instance, response):
        """Evaluate if a response is valid against the JSON schema.

        Args:
            instance (JSONSchemaBenchInstance): The JSON schema instance being evaluated.
            response (str): The model's response text.

        Returns:
            (EvaluationResult): Evaluation result for whether the response is valid against the JSON schema.
        """
        try:
            json_object = json.loads(response)
        except json.JSONDecodeError:
            return EvaluationResult(score=0, desc="invalid json")
        is_valid = validate_json_object(json_object, instance.json_schema)
        return EvaluationResult(
            score=int(is_valid),
            desc="valid" if is_valid else "json does not match schema",
        )

evaluate_sample(instance, response)

Evaluate if a response is valid against the JSON schema.

Parameters:

Name Type Description Default
instance JSONSchemaBenchInstance

The JSON schema instance being evaluated.

 required
response str

The model's response text.

 required

Returns:

Type Description
EvaluationResult

Evaluation result for whether the response is valid against the JSON schema.
Source code in genlm/eval/domains/json_schema/evaluator.py 
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def evaluate_sample(self, instance, response):
    """Evaluate if a response is valid against the JSON schema.

    Args:
        instance (JSONSchemaBenchInstance): The JSON schema instance being evaluated.
        response (str): The model's response text.

    Returns:
        (EvaluationResult): Evaluation result for whether the response is valid against the JSON schema.
    """
    try:
        json_object = json.loads(response)
    except json.JSONDecodeError:
        return EvaluationResult(score=0, desc="invalid json")
    is_valid = validate_json_object(json_object, instance.json_schema)
    return EvaluationResult(
        score=int(is_valid),
        desc="valid" if is_valid else "json does not match schema",
    )