No-error execution backend for the runtime potential. Runs generated code
against the public test inputs and reports only whether it raised; outputs are
ignored, since a wrong answer is not an error. It mirrors the vendored run_test
(call_method for stdin, Solution()/method for functional) and runs in a forked
child, since run_test patches the interpreter and needs signal.alarm.
mp_context()
Context for execution children. Prefers forkserver (forks from a clean
single-threaded server, avoiding the post-fork deadlock when the torch/vllm
parent is multi-threaded); falls back to fork where __main__ has no
file (notebooks/-c). Override with LCB_MP_METHOD.
Source code in genlm/eval/domains/livecodebench/runtime_execution.py
| def mp_context():
"""Context for execution children. Prefers ``forkserver`` (forks from a clean
single-threaded server, avoiding the post-fork deadlock when the torch/vllm
parent is multi-threaded); falls back to ``fork`` where ``__main__`` has no
file (notebooks/-c). Override with ``LCB_MP_METHOD``."""
global _MP_CTX
if _MP_CTX is not None:
return _MP_CTX
forced = os.environ.get("LCB_MP_METHOD")
if forced:
_MP_CTX = multiprocessing.get_context(forced)
return _MP_CTX
main_file = getattr(sys.modules.get("__main__"), "__file__", None)
if main_file and os.path.exists(main_file):
try:
ctx = multiprocessing.get_context("forkserver")
ctx.set_forkserver_preload(
["genlm.eval.domains.livecodebench.vendored.testing_util"])
_MP_CTX = ctx
return _MP_CTX
except Exception: # noqa: BLE001
pass
_MP_CTX = multiprocessing.get_context("fork")
return _MP_CTX
|
drop_trailing_compound(code)
Drop a trailing growable compound statement (def/for/if/class/...): its
suite can still gain lines, so a partial prefix must not execute it. Returns
None when code does not parse (caller treats that as a syntax verdict).
Source code in genlm/eval/domains/livecodebench/runtime_execution.py
| def drop_trailing_compound(code: str) -> Optional[str]:
"""Drop a trailing growable compound statement (def/for/if/class/...): its
suite can still gain lines, so a partial prefix must not execute it. Returns
None when ``code`` does not parse (caller treats that as a syntax verdict)."""
try:
tree = ast.parse(code)
except SyntaxError:
return None
if tree.body and hasattr(tree.body[-1], "body"):
tree.body = tree.body[:-1]
return ast.unparse(tree)
|
run_noerror_check(code, inputs, fn_name=None, drop_trailing_compound=False, timeout=6.0, max_total_seconds=None)
Run code against inputs in a forked child, checking only for
runtime/syntax errors (outputs are ignored). Returns one of
OK/SYNTAX/RUNTIME/TIMEOUT; TIMEOUT also covers a hung or
crashed child (the global budget firing).
Source code in genlm/eval/domains/livecodebench/runtime_execution.py
| def run_noerror_check(code: str, inputs: List[str], fn_name: Optional[str] = None,
drop_trailing_compound: bool = False, timeout: float = 6.0,
max_total_seconds: Optional[float] = None) -> str:
"""Run ``code`` against ``inputs`` in a forked child, checking only for
runtime/syntax errors (outputs are ignored). Returns one of
``OK``/``SYNTAX``/``RUNTIME``/``TIMEOUT``; ``TIMEOUT`` also covers a hung or
crashed child (the global budget firing)."""
run_timeout = max(1, math.ceil(timeout)) # signal.alarm needs an int; never round down
budget = (timeout + 1) * max(1, len(inputs)) + 5 # official lcb_runner per-sample budget
if max_total_seconds is not None:
budget = min(budget, max_total_seconds)
ctx = mp_context()
parent_conn, child_conn = ctx.Pipe(duplex=False)
p = ctx.Process(
target=_child,
args=(code, list(inputs), fn_name, drop_trailing_compound, run_timeout, child_conn),
)
p.start()
child_conn.close() # keep only the child's handle open on the write end
verdict = TIMEOUT
try:
# Wait (bounded) for the child to exit before reading, so we never call
# recv() on a live child and block forever on a partial frame.
p.join(budget)
if p.is_alive():
p.kill()
p.join()
elif parent_conn.poll(0):
try:
verdict = parent_conn.recv()
except EOFError:
verdict = RUNTIME
else:
verdict = RUNTIME # exited without sending (hard crash)
finally:
parent_conn.close()
return verdict
|