livecodebench

genlm.eval.domains.livecodebench

LiveCodeBenchDataset

Bases: Dataset[LiveCodeBenchInstance]

Dataset for LiveCodeBench evaluation (code_generation_lite).

Source code in genlm/eval/domains/livecodebench/livecodebench.py

class LiveCodeBenchDataset(Dataset[LiveCodeBenchInstance]):
    """Dataset for LiveCodeBench evaluation (code_generation_lite)."""

    def __init__(self, rows: List[Mapping[str, Any]]):
        self._rows = list(rows)

    def __len__(self) -> int:
        return len(self._rows)

    def __iter__(self) -> Iterator[LiveCodeBenchInstance]:
        for row in self._rows:
            yield LiveCodeBenchInstance(
                instance_id=row["question_id"],
                question_content=row.get("question_content", ""),
                starter_code=row.get("starter_code") or "",
                difficulty=row.get("difficulty", "unknown"),
                platform=row.get("platform", "unknown"),
                testtype=row.get("testtype", "stdin"),
                contest_date=row.get("contest_date") or "",
                eval_sample=row.get("eval_sample") or {},
                public_eval_sample=row.get("public_eval_sample") or {},
            )

    @property
    def schema(self) -> Type[LiveCodeBenchInstance]:
        return LiveCodeBenchInstance

    def to_jsonl(self, path) -> None:
        """Write the (already filtered/split) rows as a snapshot JSONL.

        The recommended way to build offline snapshots: load with ``from_hf``,
        then ``to_jsonl`` — so the snapshot inherits the date window and reloading
        it with ``from_jsonl`` (which applies no window by default) matches."""
        with Path(path).open("w") as f:
            for row in self._rows:
                f.write(json.dumps(row) + "\n")

    @classmethod
    def _build(cls, rows, start_date, end_date, difficulties, testtypes,
               holdout, test_frac, seed, shuffle, max_instances):
        rows = _filter_rows(rows, start_date, end_date, difficulties, testtypes)
        rows = _holdout_split(rows, holdout=holdout, test_frac=test_frac, seed=seed)
        if shuffle:
            random.Random(seed).shuffle(rows)
        if isinstance(max_instances, int) and max_instances >= 0:
            rows = rows[:max_instances]
        return cls(rows)

    @classmethod
    def from_hf(
        cls,
        release: str = "release_v6",
        start_date: Optional[str] = "2024-01-01",
        end_date: Optional[str] = None,
        difficulties: Optional[Sequence[str]] = None,
        testtypes: Optional[Sequence[str]] = None,
        holdout: Optional[str] = None,
        test_frac: float = 0.3,
        seed: int = 12345,
        shuffle: bool = False,
        max_instances: Optional[int] = None,
        max_tests_per_problem: Optional[int] = None,
        cumulative: bool = True,
        cache_dir: Optional[str] = None,
    ) -> "LiveCodeBenchDataset":
        """Load+decode ``livecodebench/code_generation_lite`` (needs internet / HF cache).

        ``cumulative=True`` = official version_tag semantics (all problems through
        ``release``). Defaults to the full benchmark (``holdout=None``) with
        ``contest_date >= start_date`` (2024-01-01 = after the Llama-3.x cutoffs).
        ``holdout='train'``/``'test'`` selects a stratified random partition (NOT the
        HF split name) — leave it None for leaderboard-comparable numbers."""
        start_dt, end_dt = _parse_window(start_date, end_date)
        rows = list(iter_release_rows(
            release, max_tests=max_tests_per_problem, cache_dir=cache_dir,
            cumulative=cumulative,
            # prefilter on the raw row so out-of-window private tests are never decoded
            raw_filter=lambda raw: _in_window(raw.get("contest_date"), start_dt, end_dt),
        ))
        return cls._build(rows, start_date, end_date, difficulties, testtypes,
                          holdout, test_frac, seed, shuffle, max_instances)

    @classmethod
    def from_jsonl(
        cls,
        path,
        start_date: Optional[str] = None,
        end_date: Optional[str] = None,
        difficulties: Optional[Sequence[str]] = None,
        testtypes: Optional[Sequence[str]] = None,
        holdout: Optional[str] = None,
        test_frac: float = 0.3,
        seed: int = 12345,
        shuffle: bool = False,
        max_instances: Optional[int] = None,
    ) -> "LiveCodeBenchDataset":
        """Load a snapshot JSONL written by ``to_jsonl`` (offline-friendly).

        Unlike ``from_hf``, applies NO date window by default: the snapshot is
        taken as-is (it already carries the window it was built with)."""
        with Path(path).open() as f:
            rows = [json.loads(line) for line in f if line.strip()]
        return cls._build(rows, start_date, end_date, difficulties, testtypes,
                          holdout, test_frac, seed, shuffle, max_instances)

to_jsonl(path)

Write the (already filtered/split) rows as a snapshot JSONL.

The recommended way to build offline snapshots: load with from_hf, then to_jsonl — so the snapshot inherits the date window and reloading it with from_jsonl (which applies no window by default) matches.

Source code in genlm/eval/domains/livecodebench/livecodebench.py

def to_jsonl(self, path) -> None:
    """Write the (already filtered/split) rows as a snapshot JSONL.

    The recommended way to build offline snapshots: load with ``from_hf``,
    then ``to_jsonl`` — so the snapshot inherits the date window and reloading
    it with ``from_jsonl`` (which applies no window by default) matches."""
    with Path(path).open("w") as f:
        for row in self._rows:
            f.write(json.dumps(row) + "\n")

from_hf(release='release_v6', start_date='2024-01-01', end_date=None, difficulties=None, testtypes=None, holdout=None, test_frac=0.3, seed=12345, shuffle=False, max_instances=None, max_tests_per_problem=None, cumulative=True, cache_dir=None) classmethod

Load+decode livecodebench/code_generation_lite (needs internet / HF cache).

cumulative=True = official version_tag semantics (all problems through release). Defaults to the full benchmark (holdout=None) with contest_date >= start_date (2024-01-01 = after the Llama-3.x cutoffs). holdout='train'/'test' selects a stratified random partition (NOT the HF split name) — leave it None for leaderboard-comparable numbers.

Source code in genlm/eval/domains/livecodebench/livecodebench.py

@classmethod
def from_hf(
    cls,
    release: str = "release_v6",
    start_date: Optional[str] = "2024-01-01",
    end_date: Optional[str] = None,
    difficulties: Optional[Sequence[str]] = None,
    testtypes: Optional[Sequence[str]] = None,
    holdout: Optional[str] = None,
    test_frac: float = 0.3,
    seed: int = 12345,
    shuffle: bool = False,
    max_instances: Optional[int] = None,
    max_tests_per_problem: Optional[int] = None,
    cumulative: bool = True,
    cache_dir: Optional[str] = None,
) -> "LiveCodeBenchDataset":
    """Load+decode ``livecodebench/code_generation_lite`` (needs internet / HF cache).

    ``cumulative=True`` = official version_tag semantics (all problems through
    ``release``). Defaults to the full benchmark (``holdout=None``) with
    ``contest_date >= start_date`` (2024-01-01 = after the Llama-3.x cutoffs).
    ``holdout='train'``/``'test'`` selects a stratified random partition (NOT the
    HF split name) — leave it None for leaderboard-comparable numbers."""
    start_dt, end_dt = _parse_window(start_date, end_date)
    rows = list(iter_release_rows(
        release, max_tests=max_tests_per_problem, cache_dir=cache_dir,
        cumulative=cumulative,
        # prefilter on the raw row so out-of-window private tests are never decoded
        raw_filter=lambda raw: _in_window(raw.get("contest_date"), start_dt, end_dt),
    ))
    return cls._build(rows, start_date, end_date, difficulties, testtypes,
                      holdout, test_frac, seed, shuffle, max_instances)

from_jsonl(path, start_date=None, end_date=None, difficulties=None, testtypes=None, holdout=None, test_frac=0.3, seed=12345, shuffle=False, max_instances=None) classmethod

Load a snapshot JSONL written by to_jsonl (offline-friendly).

Unlike from_hf, applies NO date window by default: the snapshot is taken as-is (it already carries the window it was built with).

Source code in genlm/eval/domains/livecodebench/livecodebench.py

@classmethod
def from_jsonl(
    cls,
    path,
    start_date: Optional[str] = None,
    end_date: Optional[str] = None,
    difficulties: Optional[Sequence[str]] = None,
    testtypes: Optional[Sequence[str]] = None,
    holdout: Optional[str] = None,
    test_frac: float = 0.3,
    seed: int = 12345,
    shuffle: bool = False,
    max_instances: Optional[int] = None,
) -> "LiveCodeBenchDataset":
    """Load a snapshot JSONL written by ``to_jsonl`` (offline-friendly).

    Unlike ``from_hf``, applies NO date window by default: the snapshot is
    taken as-is (it already carries the window it was built with)."""
    with Path(path).open() as f:
        rows = [json.loads(line) for line in f if line.strip()]
    return cls._build(rows, start_date, end_date, difficulties, testtypes,
                      holdout, test_frac, seed, shuffle, max_instances)

LiveCodeBenchEvaluator

Bases: Evaluator[LiveCodeBenchInstance]

Runs a generation's extracted code against the problem's test cases (strict 0/1).

Results are memoized on (instance_id, extracted code) — under particle-based inference many responses are byte-identical and the harness is deterministic. max_total_seconds (optional) caps the per-sample wall-clock budget; see check_correctness.

Source code in genlm/eval/domains/livecodebench/livecodebench.py

class LiveCodeBenchEvaluator(Evaluator[LiveCodeBenchInstance]):
    """Runs a generation's extracted code against the problem's test cases (strict 0/1).

    Results are memoized on (instance_id, extracted code) — under particle-based
    inference many responses are byte-identical and the harness is deterministic.
    ``max_total_seconds`` (optional) caps the per-sample wall-clock budget; see
    ``check_correctness``."""

    def __init__(self, timeout_seconds: float = 6.0, max_log_chars: int = 4000,
                 max_total_seconds: Optional[float] = None, extraction_style: str = "generic"):
        self.timeout_seconds = float(timeout_seconds)
        self.max_log_chars = int(max_log_chars)
        self.max_total_seconds = max_total_seconds
        self.extraction_style = extraction_style  # "genericbase" for base-model generations
        self._cache: Dict[Tuple[Any, str], bool] = {}

    def _passed(self, instance: LiveCodeBenchInstance, code: str) -> bool:
        key = (instance.instance_id, code)
        if key not in self._cache:
            self._cache[key] = passed_all(instance.eval_sample, code,
                                          timeout=self.timeout_seconds,
                                          max_total_seconds=self.max_total_seconds)
        return self._cache[key]

    def evaluate_sample(self, instance: LiveCodeBenchInstance, response: str) -> EvaluationResult:
        code = extract_code(response, style=self.extraction_style)
        if not code:
            return EvaluationResult(score=0.0, desc="empty code",
                                    metadata={"question_id": instance.instance_id})
        if not instance.eval_sample or "input_output" not in instance.eval_sample:
            return EvaluationResult(score=0.0, desc="missing eval_sample",
                                    metadata={"question_id": instance.instance_id})
        ok = self._passed(instance, code)
        desc = code if len(code) <= self.max_log_chars else code[: self.max_log_chars] + "\n...[truncated]"
        return EvaluationResult(
            score=1.0 if ok else 0.0,
            desc=desc,
            metadata={"question_id": instance.instance_id,
                      "difficulty": instance.difficulty,
                      "testtype": instance.testtype},
        )

LiveCodeBenchInstance

Bases: Instance

Schema for one LiveCodeBench problem (instance_id is the question_id).

eval_sample is the harness-ready {"input_output": <json str>} payload (decoded test cases); testtype is stdin or functional.

Source code in genlm/eval/domains/livecodebench/livecodebench.py

class LiveCodeBenchInstance(Instance):
    """Schema for one LiveCodeBench problem (``instance_id`` is the question_id).

    ``eval_sample`` is the harness-ready ``{"input_output": <json str>}`` payload
    (decoded test cases); ``testtype`` is ``stdin`` or ``functional``.
    """

    question_content: str
    starter_code: str = ""
    difficulty: str = "unknown"
    platform: str = "unknown"
    testtype: str = "stdin"
    contest_date: str = ""
    # Pydantic v2 deep-copies this {} default per instance (not shared state).
    eval_sample: Dict[str, str] = {}  # may be empty for a prompts-only (generation) snapshot
    # Public (example) tests only, for the verifier potentials. Empty for older snapshots.
    public_eval_sample: Dict[str, str] = {}

default_prompt_formatter(tokenizer, instance, use_chat_format=False, style='generic', enable_thinking=None)

Build the LCB prompt for instance and return token ids.

style="generic" + use_chat_format=True = LLaMa3 lcb_runner style (chat template). style="codeqwen"/"deepseek"/"genericbase" = raw completion strings; genericbase needs the matching evaluator extraction_style. enable_thinking forwards to the chat template (Qwen3 reasoning toggle).

Source code in genlm/eval/domains/livecodebench/livecodebench.py

def default_prompt_formatter(tokenizer, instance: LiveCodeBenchInstance,
                             use_chat_format: bool = False, style: str = "generic",
                             enable_thinking: bool | None = None) -> List[int]:
    """Build the LCB prompt for ``instance`` and return token ids.

    style="generic" + use_chat_format=True = LLaMa3 lcb_runner style (chat template).
    style="codeqwen"/"deepseek"/"genericbase" = raw completion strings; genericbase
    needs the matching evaluator extraction_style.
    enable_thinking forwards to the chat template (Qwen3 reasoning toggle)."""
    row = {"question_content": instance.question_content, "starter_code": instance.starter_code}
    text = format_lcb_prompt(row, tokenizer=tokenizer, chat_template=use_chat_format, style=style,
                             enable_thinking=enable_thinking)
    if style in RAW_STYLES:
        return tokenizer.encode(text)  # raw completion string; vLLM-default specials
    # Chat template already includes the BOS; avoid a second one on re-encode.
    return tokenizer.encode(text, add_special_tokens=not use_chat_format)

check_correctness(sample, generation, timeout=6.0, debug=False, max_total_seconds=None)

Run generation against the tests in a forked child.

results is per-test True/False (or sentinel ints -1/-2/-4 on failure). max_total_seconds caps the official per-sample wall-clock budget of (timeout + 1) * n_tests + 5 — the budget only binds when generated code hangs in a way signal.alarm can't interrupt, so capping it bounds the stall from a single pathological generation without affecting normal grading.

Source code in genlm/eval/domains/livecodebench/harness.py

def check_correctness(sample: Dict[str, str], generation: str,
                      timeout: float = 6.0, debug: bool = False,
                      max_total_seconds: Optional[float] = None,
                      ) -> Tuple[List[Any], Dict[str, Any]]:
    """Run ``generation`` against the tests in a forked child.

    ``results`` is per-test ``True``/``False`` (or sentinel ints ``-1``/``-2``/``-4``
    on failure). ``max_total_seconds`` caps the official per-sample wall-clock budget
    of ``(timeout + 1) * n_tests + 5`` — the budget only binds when generated code
    hangs in a way ``signal.alarm`` can't interrupt, so capping it bounds the stall
    from a single pathological generation without affecting normal grading."""
    # Guard the input_output parse (official lcb_runner does this unguarded, but our
    # from_jsonl allows prompts-only snapshots where it may be absent/malformed): a bad
    # sample scores fail instead of crashing the whole eval run.
    try:
        n_tests = len(json.loads(sample["input_output"])["inputs"])
    except (KeyError, TypeError, ValueError):
        return [-1], {"error": "missing or malformed eval_sample"}
    run_timeout = max(1, math.ceil(timeout))  # signal.alarm needs an int; never round down
    budget = (timeout + 1) * n_tests + 5  # official lcb_runner per-sample budget
    if max_total_seconds is not None:
        budget = min(budget, max_total_seconds)
    ctx = mp_context()
    parent_conn, child_conn = ctx.Pipe(duplex=False)
    p = ctx.Process(
        target=_child_run,
        args=(sample, generation, debug, child_conn, run_timeout, capture.is_enabled()),
    )
    p.start()
    child_conn.close()  # keep only the child's handle open on the write end
    try:
        # Wait (bounded) for the child to exit before reading, so we never call
        # recv() on a live child and block forever on a partial frame.
        p.join(budget)
        if p.is_alive():
            p.kill()
            p.join()
        elif parent_conn.poll(0):
            try:
                res, metadata = parent_conn.recv()
                return list(res), dict(metadata)
            except EOFError:  # crashed mid-send
                pass
    finally:
        parent_conn.close()
    return [-1] * n_tests, {"error": "global timeout or crashed child"}

passed_all(sample, generation, timeout=6.0, max_total_seconds=None)

True iff every test passed (> 0), matching official np.all(gen > 0).

Source code in genlm/eval/domains/livecodebench/harness.py

def passed_all(sample: Dict[str, str], generation: str, timeout: float = 6.0,
               max_total_seconds: Optional[float] = None) -> bool:
    """True iff every test passed (``> 0``), matching official ``np.all(gen > 0)``."""
    results, _ = check_correctness(sample, generation, timeout=timeout,
                                   max_total_seconds=max_total_seconds)
    return bool(results) and all(r > 0 for r in results)

decode_context(context)

Decode a genlm.control context (str/bytes/list of byte tokens or int byte ids) into text.

Source code in genlm/eval/domains/livecodebench/prompts.py

def decode_context(context) -> str:
    """Decode a genlm.control context (str/bytes/list of byte tokens or int byte
    ids) into text."""
    if not context:
        return ""
    if isinstance(context, str):
        return context
    if isinstance(context, bytes):
        return context.decode("utf-8", errors="ignore")
    pieces = []
    for tok in context:
        if isinstance(tok, int):
            pieces.append(bytes([tok]))
        elif isinstance(tok, bytes):
            pieces.append(tok)
        else:
            pieces.append(str(tok).encode("utf-8", errors="ignore"))
    return b"".join(pieces).decode("utf-8", errors="ignore")

extract_code(model_output, style='generic')

Code between the last two ``` fences (last block if 3+); "" if fewer than two. style="genericbase" = whole stripped output. Matches lcb_runner extract_code.

Source code in genlm/eval/domains/livecodebench/prompts.py

def extract_code(model_output: str, style: str = "generic") -> str:
    """Code between the last two ``` fences (last block if 3+); "" if fewer than two.
    style="genericbase" = whole stripped output. Matches lcb_runner extract_code."""
    # Reasoning models (Qwen3, R1, ...) emit <think>...</think> before the answer; keep only the
    # post-think answer so a code fence inside the reasoning can't be mistaken for the solution.
    # No </think> (every existing non-reasoning model) leaves the output unchanged.
    if "</think>" in model_output:
        model_output = model_output.rsplit("</think>", 1)[1]
    if style == "genericbase":
        return model_output.strip()
    lines = model_output.split("\n")
    fence_idxs = [i for i, ln in enumerate(lines) if "```" in ln]
    if len(fence_idxs) < 2:
        return ""
    return "\n".join(lines[fence_idxs[-2] + 1: fence_idxs[-1]])

extract_code_prefix(model_output, style='generic')

Code being written, for prefix scoring: text after the last open fence, or "" when no block is open. Deferring on a closed block (a later block could supersede it) keeps prefix consistent with extract_code at complete. style="genericbase" = whole stripped output.

Source code in genlm/eval/domains/livecodebench/prompts.py

def extract_code_prefix(model_output: str, style: str = "generic") -> str:
    """Code being written, for prefix scoring: text after the last open fence, or
    "" when no block is open. Deferring on a closed block (a later block could
    supersede it) keeps prefix consistent with ``extract_code`` at complete.
    style="genericbase" = whole stripped output."""
    if style == "genericbase":
        return model_output.strip()
    lines = model_output.split("\n")
    fence_idxs = [i for i, ln in enumerate(lines) if "```" in ln]
    if len(fence_idxs) % 2 == 1:  # block open: judge text after the last fence
        return "\n".join(lines[fence_idxs[-1] + 1:])
    return ""  # closed or no block: defer to complete()

format_lcb_prompt(row, tokenizer=None, chat_template=False, style='generic', enable_thinking=None)

Prompt for an lcb_runner LMStyle: "generic" (LLaMa3, via chat template when chat_template=True), "codeqwen" (CodeQwenInstruct, raw <|im_*|> string), or "deepseek" (DeepSeekCodeInstruct, raw ### Instruction/Response string).

enable_thinking forwards to apply_chat_template (Qwen3-style reasoning toggle); left out of the call when None so non-reasoning templates are unaffected.

Source code in genlm/eval/domains/livecodebench/prompts.py

def format_lcb_prompt(row: Mapping[str, str], tokenizer=None,
                      chat_template: bool = False, style: str = "generic",
                      enable_thinking: bool | None = None) -> str:
    """Prompt for an lcb_runner LMStyle: "generic" (LLaMa3, via chat template when
    chat_template=True), "codeqwen" (CodeQwenInstruct, raw <|im_*|> string), or
    "deepseek" (DeepSeekCodeInstruct, raw ### Instruction/Response string).

    enable_thinking forwards to apply_chat_template (Qwen3-style reasoning toggle); left
    out of the call when None so non-reasoning templates are unaffected."""
    if style not in STYLES:
        raise ValueError(f"style must be one of {STYLES}; got {style!r}")
    qc, sc = row.get("question_content", ""), row.get("starter_code", "") or ""
    if style == "codeqwen":
        # official joins system + body with a blank line ("...<|im_start|>user\n\n...")
        return f"{SYSTEM_MESSAGE_CODEQWEN}\n\n{_codeqwen_body(qc, sc)}"
    if style == "deepseek":
        return f"{SYSTEM_MESSAGE_DEEPSEEK}\n\n{_deepseek_body(qc, sc)}"
    if style == "genericbase":
        return _genericbase_body(qc, sc)
    body = _user_body(qc, sc)
    if chat_template and tokenizer is not None:
        messages = [{"role": "system", "content": SYSTEM_MESSAGE},
                    {"role": "user", "content": body}]
        kw = {} if enable_thinking is None else {"enable_thinking": enable_thinking}
        return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, **kw)
    return f"{SYSTEM_MESSAGE}\n\n{body}"

build_row(raw, release, max_tests=None)

Convert a raw HF row into a clean snapshot row with a harness-ready eval_sample ({"input_output": <json str>}).

Source code in genlm/eval/domains/livecodebench/fetch.py

def build_row(raw: Mapping[str, Any], release: str, max_tests: Optional[int] = None) -> Dict[str, Any]:
    """Convert a raw HF row into a clean snapshot row with a harness-ready
    ``eval_sample`` (``{"input_output": <json str>}``)."""
    public = json.loads(raw["public_test_cases"])
    private = _decode_private(raw["private_test_cases"])
    all_tests = list(public) + list(private)
    metadata = json.loads(raw["metadata"]) if isinstance(raw.get("metadata"), str) else (raw.get("metadata") or {})

    inputs = [t["input"] for t in all_tests]
    outputs = [t["output"] for t in all_tests]
    if max_tests is not None:
        inputs, outputs = inputs[:max_tests], outputs[:max_tests]

    fn_name = metadata.get("func_name")
    eval_sample = {"input_output": json.dumps({
        "inputs": inputs, "outputs": outputs, "fn_name": fn_name,
    })}
    # Public tests, separated so verifier potentials use them without leaking the
    # private set. Public sort first, so this is a prefix of eval_sample.
    n_public = len(public) if max_tests is None else min(len(public), max_tests)
    public_eval_sample = {"input_output": json.dumps({
        "inputs": inputs[:n_public], "outputs": outputs[:n_public], "fn_name": fn_name,
    })}
    return {
        "question_id": raw.get("question_id"),
        "question_content": raw.get("question_content", ""),
        "starter_code": raw.get("starter_code", "") or "",
        "difficulty": raw.get("difficulty", "unknown"),
        "platform": raw.get("platform", "unknown"),
        "contest_date": raw.get("contest_date", ""),
        "testtype": derive_testtype(metadata, all_tests),
        "release": release,
        "eval_sample": eval_sample,
        "public_eval_sample": public_eval_sample,
        "metadata": metadata,
    }

derive_testtype(metadata, tests)

functional iff a func_name is present (matches run_test's which_type), else fall back to the first test's recorded testtype, else stdin.

Source code in genlm/eval/domains/livecodebench/fetch.py

def derive_testtype(metadata: Mapping[str, Any], tests: List[Mapping[str, Any]]) -> str:
    """functional iff a func_name is present (matches run_test's which_type),
    else fall back to the first test's recorded testtype, else stdin."""
    if metadata.get("func_name"):
        return "functional"
    if tests and tests[0].get("testtype"):
        return str(tests[0]["testtype"])
    return "stdin"

iter_release_rows(release='release_v6', max_tests=None, cache_dir=None, cumulative=True, raw_filter=None)

Yield clean built rows for a release (needs HF cache).

cumulative=True (official version_tag semantics) loads test.jsonl..testN.jsonl de-duped by question_id (release_v6 == ~1055); cumulative=False loads only that window. Dedup keeps the first occurrence, so release = first-seen.

raw_filter is applied to the raw HF row BEFORE the (expensive) private-test decode, so callers can drop e.g. out-of-window rows cheaply.

Source code in genlm/eval/domains/livecodebench/fetch.py

def iter_release_rows(release: str = "release_v6", max_tests: Optional[int] = None,
                      cache_dir: Optional[str] = None, cumulative: bool = True,
                      raw_filter: Optional[Callable[[Mapping[str, Any]], bool]] = None,
                      ) -> Iterator[Dict[str, Any]]:
    """Yield clean built rows for a release (needs HF cache).

    ``cumulative=True`` (official version_tag semantics) loads test.jsonl..testN.jsonl
    de-duped by question_id (release_v6 == ~1055); ``cumulative=False`` loads only that
    window. Dedup keeps the first occurrence, so ``release`` = first-seen.

    ``raw_filter`` is applied to the raw HF row BEFORE the (expensive) private-test
    decode, so callers can drop e.g. out-of-window rows cheaply."""
    from huggingface_hub import hf_hub_download  # lazy: only from_hf needs the extra

    n = _release_num(release)
    tags = [f"release_v{i}" for i in range(1, n + 1)] if cumulative else [release]
    seen = set()
    for tag in tags:
        path = hf_hub_download(repo_id=HF_REPO, filename=_release_filename(tag),
                               repo_type="dataset", cache_dir=cache_dir)
        with open(path) as fin:
            for line in fin:
                if not line.strip():
                    continue
                raw = json.loads(line)
                if raw.get("question_id") in seen:
                    continue
                if raw_filter is not None and not raw_filter(raw):
                    continue
                row = build_row(raw, release=tag, max_tests=max_tests)
                seen.add(row["question_id"])
                yield row

LCBRuntimeNoErrorPotential

Bases: Potential

0.0 if the extracted code runs without error, -inf otherwise; wrong answers are tolerated. Uses only the public test inputs, never the held-out private ones.

Source code in genlm/eval/domains/livecodebench/runtime_no_error_potential.py

class LCBRuntimeNoErrorPotential(Potential):
    """0.0 if the extracted code runs without error, -inf otherwise; wrong
    answers are tolerated. Uses only the public test inputs, never the held-out
    private ones."""

    def __init__(
        self,
        vocabulary=None,
        public_eval_sample: Optional[dict] = None,
        timeout_seconds: float = 6.0,
        max_inputs: int = 1,
        extraction_style: str = "generic",
        f: Optional[Callable[[List[bytes]], List[bytes]]] = None,
    ):
        vocabulary = vocabulary or [bytes([i]) for i in range(256)]
        super().__init__(vocabulary=vocabulary)
        self.public_eval_sample = public_eval_sample or {}
        self.timeout_seconds = float(timeout_seconds)
        self.max_inputs = int(max_inputs)
        self.extraction_style = extraction_style
        self.f = f
        self.last_was_syntax_error = False

        io = {}
        if self.public_eval_sample.get("input_output"):
            try:
                io = json.loads(self.public_eval_sample["input_output"])
            except (TypeError, ValueError):
                io = {}
        # Public inputs only, no outputs, so nothing leaks. prefix() uses a
        # subset of complete()'s inputs, which keeps the soundness invariant.
        self._inputs = list(io.get("inputs") or [])
        self._prefix_inputs = self._inputs[: self.max_inputs]
        self._fn_name = io.get("fn_name")

        # Prefixes extending a hung one re-run the same statements: defer them.
        self._hung_prefixes: List[str] = []
        self._hung_prefixes_maxsize = 16
        # SMC clones particles into repeated prefixes; cache verdicts per code.
        self._score_cache: OrderedDict = OrderedDict()
        self._score_cache_maxsize = 4096
        self.cache_hits = 0
        self.cache_misses = 0

    def coerce(self, other, f=None, prune=True):
        return LCBRuntimeNoErrorPotential(
            vocabulary=list(other.vocab),
            public_eval_sample=self.public_eval_sample,
            timeout_seconds=self.timeout_seconds,
            max_inputs=self.max_inputs,
            extraction_style=self.extraction_style,
            f=f,
        )

    async def prefix(self, context: List[bytes]) -> float:
        if self.f is not None:
            context = self.f(context)
        raw = decode_context(context)
        # Newline guardrail: only judge at line boundaries (default line sampler).
        if not raw.endswith("\n"):
            return 0.0
        code = extract_code_prefix(raw, self.extraction_style)
        if not code.strip():
            self.last_was_syntax_error = False
            return 0.0
        status = _syntax_status(code)
        if status == "incomplete":
            self.last_was_syntax_error = False
            return 0.0
        if status == "broken":
            self.last_was_syntax_error = True
            return float("-inf")
        if self._fn_name:
            # Functional: only complete() calls the entrypoint. Calling it on a
            # prefix is unsound (a helper it needs may be defined later, then dropped).
            self.last_was_syntax_error = False
            return 0.0
        if not self._prefix_inputs:  # syntax-only mode: parseable prefix, nothing to run
            self.last_was_syntax_error = False
            return 0.0
        for hung in self._hung_prefixes:
            if code.startswith(hung):
                return 0.0
        return await self._run(code, mode="prefix")

    async def complete(self, context: List[bytes]) -> float:
        if self.f is not None:
            context = self.f(context)
        code = extract_code(decode_context(context), self.extraction_style)
        if not code.strip():
            self.last_was_syntax_error = False
            return float("-inf")  # an empty completion is not a runnable solution
        status = _syntax_status(code)
        if status in ("broken", "incomplete"):  # at EOS, unparseable code is fatal
            self.last_was_syntax_error = True
            return float("-inf")
        if not self._inputs:
            self.last_was_syntax_error = False
            return 0.0
        return await self._run(code, mode="complete")

    async def _run(self, code: str, mode: str) -> float:
        key = (mode, code)
        cached = self._score_cache.get(key)
        if cached is not None:
            self._score_cache.move_to_end(key)
            self.cache_hits += 1
            value, syntax_error = cached
            self.last_was_syntax_error = syntax_error
            return value
        self.cache_misses += 1

        # prefix: a capped input subset (cheap); complete: all (a later input may crash).
        inputs = self._prefix_inputs if mode == "prefix" else self._inputs
        async with fork_semaphore():
            verdict = await asyncio.to_thread(
                run_noerror_check, code, inputs, self._fn_name,
                mode == "prefix", self.timeout_seconds,
            )

        if verdict == TIMEOUT:
            # A slow prefix may still finish: defer (uncached). complete kills.
            self.last_was_syntax_error = False
            if mode == "prefix":
                self._hung_prefixes.append(code)
                if len(self._hung_prefixes) > self._hung_prefixes_maxsize:
                    self._hung_prefixes.pop(0)
                return 0.0
            return float("-inf")

        self.last_was_syntax_error = verdict == SYNTAX
        value = 0.0 if verdict == OK else float("-inf")
        self._score_cache[key] = (value, self.last_was_syntax_error)
        self._score_cache.move_to_end(key)
        if len(self._score_cache) > self._score_cache_maxsize:
            self._score_cache.popitem(last=False)
        return value

LCBPublicTestPotential

Bases: Potential

Soft public-test verifier. prefix never kills; complete returns 0.0 when all public tests pass, otherwise a finite penalty proportional to the number of failed tests (floored at min_score, never -inf).

Source code in genlm/eval/domains/livecodebench/public_test_potential.py

class LCBPublicTestPotential(Potential):
    """Soft public-test verifier. ``prefix`` never kills; ``complete`` returns
    0.0 when all public tests pass, otherwise a finite penalty proportional to
    the number of failed tests (floored at ``min_score``, never -inf)."""

    def __init__(
        self,
        vocabulary=None,
        public_eval_sample: Optional[dict] = None,
        timeout_seconds: float = 6.0,
        penalty_per_failed: float = 2.0,
        min_score: float = -10.0,
        extraction_style: str = "generic",
        max_total_seconds: Optional[float] = None,
        f: Optional[Callable[[List[bytes]], List[bytes]]] = None,
    ):
        vocabulary = vocabulary or [bytes([i]) for i in range(256)]
        super().__init__(vocabulary=vocabulary)
        self.public_eval_sample = public_eval_sample or {}
        self.timeout_seconds = float(timeout_seconds)
        self.penalty_per_failed = abs(float(penalty_per_failed))
        self.min_score = float(min_score)
        self.extraction_style = extraction_style
        self.max_total_seconds = max_total_seconds
        self.f = f

        io = {}
        if self.public_eval_sample.get("input_output"):
            try:
                io = json.loads(self.public_eval_sample["input_output"])
            except (TypeError, ValueError):
                io = {}
        self._inputs = list(io.get("inputs") or [])
        self._outputs = list(io.get("outputs") or [])
        self._fn_name = io.get("fn_name")
        self._cache: OrderedDict = OrderedDict()
        self._cache_maxsize = 2048

    def coerce(self, other, f=None, prune=True):
        return LCBPublicTestPotential(
            vocabulary=list(other.vocab),
            public_eval_sample=self.public_eval_sample,
            timeout_seconds=self.timeout_seconds,
            penalty_per_failed=self.penalty_per_failed,
            min_score=self.min_score,
            extraction_style=self.extraction_style,
            max_total_seconds=self.max_total_seconds,
            f=f,
        )

    async def prefix(self, context: List[bytes]) -> float:
        return 0.0  # runs only at the end of generation; never kills a prefix

    async def complete(self, context: List[bytes]) -> float:
        if self.f is not None:
            context = self.f(context)
        code = self._extract(decode_context(context))
        if not self._inputs:
            return 0.0
        feedback = self._cache.get(code)
        if feedback is None:
            # Fork off the event loop, bounded; cache writes stay in the
            # coroutine so concurrent particles don't race on the dict.
            async with fork_semaphore():
                feedback = await asyncio.to_thread(self._evaluate, code)
            if not feedback.transient:
                self._store(code, feedback)
        else:
            self._cache.move_to_end(code)
        return self._score(feedback)

    def _score(self, feedback: "PublicTestFeedback") -> float:
        if feedback.n_public == 0 or feedback.all_passed:
            return 0.0
        return max(self.min_score, -self.penalty_per_failed * feedback.n_failed)

    def run_public_tests(self, generation: str) -> PublicTestFeedback:
        """Run every public test on ``generation`` (a full model output; code is
        extracted with the configured style) and return structured feedback.
        Results are cached per extracted code."""
        if not self._inputs:
            return PublicTestFeedback(n_public=0, n_passed=0)
        code = self._extract(generation)
        feedback = self._cache.get(code)
        if feedback is None:
            feedback = self._evaluate(code)
            if not feedback.transient:
                self._store(code, feedback)
        else:
            self._cache.move_to_end(code)
        return feedback

    def _extract(self, generation: str) -> str:
        # Extract exactly as the evaluator (no fallback for unfenced output).
        return extract_code(generation, self.extraction_style)

    def _store(self, code: str, feedback: "PublicTestFeedback") -> None:
        self._cache[code] = feedback
        self._cache.move_to_end(code)
        if len(self._cache) > self._cache_maxsize:
            self._cache.popitem(last=False)

    def _evaluate(self, code: str) -> PublicTestFeedback:
        # One test per run so a failure does not hide later ones (the grader
        # short-circuits within a sample on the first miss).
        results: List[PublicTestResult] = []
        n_passed, transient = 0, False
        for i, (inp, out) in enumerate(zip(self._inputs, self._outputs)):
            sample = {"input_output": json.dumps(
                {"inputs": [inp], "outputs": [out], "fn_name": self._fn_name})}
            res, meta = check_correctness(
                sample, code, timeout=self.timeout_seconds,
                max_total_seconds=self.max_total_seconds)
            transient = transient or (-1 in res)  # global timeout / crash, not the code
            passed = bool(res) and all(r > 0 for r in res)
            n_passed += int(passed)
            results.append(PublicTestResult(
                index=i, input=inp, expected=out, passed=passed,
                error_message="" if passed else str(
                    meta.get("error_message") or meta.get("error") or ""),
                got="" if passed else str(meta.get("output", "")),
            ))
        return PublicTestFeedback(
            n_public=len(results), n_passed=n_passed, results=results,
            transient=transient)

run_public_tests(generation)

Run every public test on generation (a full model output; code is extracted with the configured style) and return structured feedback. Results are cached per extracted code.

Source code in genlm/eval/domains/livecodebench/public_test_potential.py

def run_public_tests(self, generation: str) -> PublicTestFeedback:
    """Run every public test on ``generation`` (a full model output; code is
    extracted with the configured style) and return structured feedback.
    Results are cached per extracted code."""
    if not self._inputs:
        return PublicTestFeedback(n_public=0, n_passed=0)
    code = self._extract(generation)
    feedback = self._cache.get(code)
    if feedback is None:
        feedback = self._evaluate(code)
        if not feedback.transient:
            self._store(code, feedback)
    else:
        self._cache.move_to_end(code)
    return feedback

PublicTestFeedback dataclass

Aggregate public-test outcome for a generation, with a repair summary.

Source code in genlm/eval/domains/livecodebench/public_test_potential.py

@dataclass
class PublicTestFeedback:
    """Aggregate public-test outcome for a generation, with a repair summary."""

    n_public: int
    n_passed: int
    results: List[PublicTestResult] = field(default_factory=list)
    # Global timeout / crash (overload, not the code): not cached, so retries re-run.
    transient: bool = False

    @property
    def all_passed(self) -> bool:
        return self.n_public > 0 and self.n_passed == self.n_public

    @property
    def n_failed(self) -> int:
        return self.n_public - self.n_passed

    @property
    def pass_fraction(self) -> float:
        return self.n_passed / self.n_public if self.n_public else 1.0

    def summary(self, max_cases: int = 3, max_chars: int = 300) -> str:
        """Human-readable failing-test report for a repair prompt."""
        if self.n_public == 0:
            return "No public tests were available."
        if self.all_passed:
            return f"All {self.n_public} public tests passed."

        def trim(s: str) -> str:
            s = str(s)
            return s if len(s) <= max_chars else s[:max_chars] + "...[truncated]"

        lines = [f"Passed {self.n_passed}/{self.n_public} public tests. Failing cases:"]
        shown = [r for r in self.results if not r.passed][:max_cases]
        for r in shown:
            lines.append(f"- Input:\n{trim(r.input)}")
            lines.append(f"  Expected output:\n{trim(r.expected)}")
            if r.got:
                lines.append(f"  Your output:\n{trim(r.got)}")
            if r.error_message:
                lines.append(f"  Error: {trim(r.error_message)}")
        return "\n".join(lines)

summary(max_cases=3, max_chars=300)

Human-readable failing-test report for a repair prompt.

Source code in genlm/eval/domains/livecodebench/public_test_potential.py

def summary(self, max_cases: int = 3, max_chars: int = 300) -> str:
    """Human-readable failing-test report for a repair prompt."""
    if self.n_public == 0:
        return "No public tests were available."
    if self.all_passed:
        return f"All {self.n_public} public tests passed."

    def trim(s: str) -> str:
        s = str(s)
        return s if len(s) <= max_chars else s[:max_chars] + "...[truncated]"

    lines = [f"Passed {self.n_passed}/{self.n_public} public tests. Failing cases:"]
    shown = [r for r in self.results if not r.passed][:max_cases]
    for r in shown:
        lines.append(f"- Input:\n{trim(r.input)}")
        lines.append(f"  Expected output:\n{trim(r.expected)}")
        if r.got:
            lines.append(f"  Your output:\n{trim(r.got)}")
        if r.error_message:
            lines.append(f"  Error: {trim(r.error_message)}")
    return "\n".join(lines)

PublicTestResult dataclass

Outcome of running a single public test.

Source code in genlm/eval/domains/livecodebench/public_test_potential.py

@dataclass
class PublicTestResult:
    """Outcome of running a single public test."""

    index: int
    input: str
    expected: str
    passed: bool
    error_message: str = ""
    got: str = ""

format_repair_prompt(tokenizer, instance, previous_generation, feedback, use_chat_format=False, style='generic')

Build the next-turn repair prompt for instance and return token ids, matching the contract of default_prompt_formatter.

Source code in genlm/eval/domains/livecodebench/public_test_potential.py

def format_repair_prompt(tokenizer, instance, previous_generation: str,
                         feedback: PublicTestFeedback, use_chat_format: bool = False,
                         style: str = "generic"):
    """Build the next-turn repair prompt for ``instance`` and return token ids,
    matching the contract of ``default_prompt_formatter``."""
    previous_code = extract_code(previous_generation, style)
    question = repair_question_content(instance.question_content, previous_code, feedback)
    row = {"question_content": question, "starter_code": instance.starter_code}
    text = format_lcb_prompt(row, tokenizer=tokenizer, chat_template=use_chat_format, style=style)
    from genlm.eval.domains.livecodebench.prompts import RAW_STYLES

    if style in RAW_STYLES:
        return tokenizer.encode(text)
    return tokenizer.encode(text, add_special_tokens=not use_chat_format)

repair_question_content(question_content, previous_code, feedback)

Augment the original question with the failed attempt and public-test feedback, for a second (repair) generation turn.

Source code in genlm/eval/domains/livecodebench/public_test_potential.py

def repair_question_content(question_content: str, previous_code: str,
                            feedback: PublicTestFeedback) -> str:
    """Augment the original question with the failed attempt and public-test
    feedback, for a second (repair) generation turn."""
    return (
        f"{question_content}\n\n"
        f"A previous attempt produced this program:\n"
        f"```python\n{previous_code}\n```\n\n"
        f"{feedback.summary()}\n\n"
        f"Fix the program so it passes all tests."
    )