Skip to content

capture

genlm.eval.domains.livecodebench.capture

Full per-test execution capture for LiveCodeBench. run_test calls grade_call_based / grade_stdio as bare module-level names (testing_util.py lines 476/497), so enable_capture() swaps in instrumented copies that:

  • run every test (never short-circuit at the first wrong answer),
  • record each test's input / expected / actual output untruncated (no truncatefn),
  • stop only on a hard fault (-3 timeout / -4 runtime error), where the interpreter state after a signal-driven exception is unsafe to keep running,
  • carry the per-test records back out through the returned metadata["executions"] (which already flows to the parent via the harness pipe).

The official pass/fail is unchanged: passed_all is all(r > 0 for r in results).

grade_call_based_cap(code, all_inputs, all_outputs, fn_name, timeout)

Instrumented grade_call_based: runs all tests, captures every return value.

Source code in genlm/eval/domains/livecodebench/capture.py 
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
def grade_call_based_cap(code, all_inputs, all_outputs, fn_name, timeout):
    """Instrumented ``grade_call_based``: runs all tests, captures every return value."""
    code = import_string + "\n\n" + code
    compiled_sol = compile_code(code, timeout)
    if compiled_sol is None:
        return [COMPILE], {"executions": [], "n_tests": len(all_outputs), "error_code": COMPILE}
    method = get_function(compiled_sol, fn_name)
    if method is None:
        return [COMPILE], {"executions": [], "n_tests": len(all_outputs), "error_code": COMPILE}

    all_inputs = [[json.loads(line) for line in inputs.split("\n")] for inputs in all_inputs]
    all_outputs = [json.loads(output) for output in all_outputs]

    results, records, total = [], [], 0.0
    for idx, (gt_inp, gt_out) in enumerate(zip(all_inputs, all_outputs)):
        signal.alarm(timeout)
        faulthandler.enable()
        try:
            start = time.time()
            prediction = method(*gt_inp)
            dt = time.time() - start
            total += dt
            signal.alarm(0)
            if isinstance(prediction, tuple):
                prediction = list(prediction)
            ok = prediction == gt_out
            results.append(bool(ok))
            records.append(_rec(idx, "return", gt_inp, gt_out, prediction,
                                ok, OK if ok else WRONG, None if ok else "Wrong Answer", dt))
        except Exception as e:  # noqa: BLE001 (mirror vendored broad catch)
            signal.alarm(0)
            tle = "timeoutexception" in repr(e).lower()
            code_e = TLE if tle else RTE
            results.append(code_e)
            records.append(_rec(idx, "return", gt_inp, gt_out, None, False, code_e,
                                ("Time Limit Exceeded: " if tle else "Runtime Error: ") + repr(e), 0.0))
            faulthandler.disable()
            break  # interpreter state after a signal-driven exception is unsafe
        finally:
            signal.alarm(0)
            faulthandler.disable()

    return results, {"executions": records, "n_tests": len(all_outputs), "execution_time": total}

grade_stdio_cap(code, all_inputs, all_outputs, timeout)

Instrumented grade_stdio: runs all tests, captures every stdout untruncated.

Source code in genlm/eval/domains/livecodebench/capture.py 
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
def grade_stdio_cap(code, all_inputs, all_outputs, timeout):
    """Instrumented ``grade_stdio``: runs all tests, captures every stdout untruncated."""
    code = clean_if_name(code)
    code = make_function(code)
    compiled_sol = compile_code(code, timeout)
    if compiled_sol is None:
        return [COMPILE], {"executions": [], "n_tests": len(all_outputs), "error_code": COMPILE}
    method = get_function(compiled_sol, "wrapped_function")
    if method is None:
        return [COMPILE], {"executions": [], "n_tests": len(all_outputs), "error_code": COMPILE}

    results, records, total = [], [], 0.0
    for idx, (gt_inp, gt_out) in enumerate(zip(all_inputs, all_outputs)):
        signal.alarm(timeout)
        faulthandler.enable()
        with Capturing() as captured_output:
            try:
                start = time.time()
                call_method(method, gt_inp)
                dt = time.time() - start
                signal.alarm(0)
            except Exception as e:  # noqa: BLE001
                signal.alarm(0)
                tle = "timeoutexception" in repr(e).lower()
                code_e = TLE if tle else RTE
                results.append(code_e)
                # captured_output[0] is set on __exit__; read it after the with-block
                _err = ("Time Limit Exceeded: " if tle else "Runtime Error: ") + repr(e)
                faulthandler.disable()
                _broke = (idx, gt_inp, gt_out, code_e, _err)
                break
            finally:
                signal.alarm(0)
                faulthandler.disable()
        prediction = captured_output[0]
        ok, msg = _stdio_match(prediction, gt_out)
        results.append(bool(ok))
        records.append(_rec(idx, "stdio", gt_inp, gt_out, prediction,
                            ok, OK if ok else WRONG, None if ok else msg, dt))
    else:
        return results, {"executions": records, "n_tests": len(all_outputs), "execution_time": total}

    # loop broke on a hard fault: record it (stdout captured up to the fault)
    idx, gt_inp, gt_out, code_e, _err = _broke
    records.append(_rec(idx, "stdio", gt_inp, gt_out, (captured_output[0] if captured_output else ""),
                        False, code_e, _err, 0.0))
    return results, {"executions": records, "n_tests": len(all_outputs), "execution_time": total}

enable_capture()

Swap the vendored grade functions for the capturing copies (idempotent).

The harness child re-enables capture itself, so this need only be set in the parent before grading.

Source code in genlm/eval/domains/livecodebench/capture.py 
191
192
193
194
195
196
197
198
199
200
201
202
203
def enable_capture():
    """Swap the vendored grade functions for the capturing copies (idempotent).

    The harness child re-enables capture itself, so this need only be set in the
    parent before grading.
    """
    global _ENABLED
    # drift guard: the vendored originals must still be the functions we mirrored
    assert hasattr(_tu, "grade_call_based") and hasattr(_tu, "grade_stdio"), \
        "vendored testing_util changed: grade_call_based/grade_stdio missing"
    _tu.grade_call_based = grade_call_based_cap
    _tu.grade_stdio = grade_stdio_cap
    _ENABLED = True

disable_capture()

Restore the vendored grade functions and clear the flag (inverse of enable_capture).

Source code in genlm/eval/domains/livecodebench/capture.py 
206
207
208
209
210
def disable_capture():
    """Restore the vendored grade functions and clear the flag (inverse of enable_capture)."""
    global _ENABLED
    _tu.grade_call_based, _tu.grade_stdio = _ORIGINALS
    _ENABLED = False