livecodebench_multilingual

genlm.eval.domains.livecodebench_multilingual

MultilingualLCBDataset

Bases: Dataset[MultilingualLCBInstance]

LiveCodeBench stdin problems for a single target language.

Source code in genlm/eval/domains/livecodebench_multilingual/dataset.py

class MultilingualLCBDataset(Dataset[MultilingualLCBInstance]):
    """LiveCodeBench stdin problems for a single target language."""

    def __init__(self, rows: List[Mapping[str, Any]], language: str):
        self.language = resolve_language(language).key
        self._rows = list(rows)

    def __len__(self) -> int:
        return len(self._rows)

    def __iter__(self) -> Iterator[MultilingualLCBInstance]:
        for row in self._rows:
            if row.get("question_id") is None:
                raise ValueError("LiveCodeBench row is missing question_id")
            qid = str(row["question_id"])
            yield MultilingualLCBInstance(
                instance_id=f"{qid}@{self.language}",
                question_id=qid,
                language=self.language,
                question_content=row.get("question_content", ""),
                starter_code=row.get("starter_code") or "",
                difficulty=row.get("difficulty", "unknown"),
                platform=row.get("platform", "unknown"),
                testtype=row.get("testtype", "stdin"),
                contest_date=row.get("contest_date") or "",
                eval_sample=row.get("eval_sample") or {},
            )

    @property
    def schema(self) -> Type[MultilingualLCBInstance]:
        return MultilingualLCBInstance

    def to_jsonl(self, path) -> None:
        """Write the stdin rows as a language-independent snapshot (language is a per-run tag,
        not stored; reload with ``from_jsonl(path, language=...)``)."""
        LiveCodeBenchDataset(self._rows).to_jsonl(path)

    @classmethod
    def from_hf(
        cls,
        language: str,
        *,
        release: str = "release_v6",
        start_date: Optional[str] = "2024-01-01",
        end_date: Optional[str] = None,
        difficulties: Optional[Sequence[str]] = None,
        holdout: Optional[str] = None,
        test_frac: float = 0.3,
        seed: int = 12345,
        shuffle: bool = False,
        max_instances: Optional[int] = None,
        max_tests_per_problem: Optional[int] = None,
        cumulative: bool = True,
        cache_dir: Optional[str] = None,
    ) -> "MultilingualLCBDataset":
        """Load stdin LiveCodeBench problems for ``language`` (testtypes forced to stdin).

        ``start_date`` defaults to the base loader's ``2024-01-01``; pass the paper's window
        (e.g. ``None`` or ``'2024-07-01'``) to match a specific problem set.
        """
        resolve_language(language)  # validate early
        base = LiveCodeBenchDataset.from_hf(
            release=release,
            start_date=start_date,
            end_date=end_date,
            difficulties=difficulties,
            testtypes=_STDIN_ONLY,
            holdout=holdout,
            test_frac=test_frac,
            seed=seed,
            shuffle=shuffle,
            max_instances=max_instances,
            max_tests_per_problem=max_tests_per_problem,
            cumulative=cumulative,
            cache_dir=cache_dir,
        )
        return cls(base._rows, language)

    @classmethod
    def from_jsonl(
        cls,
        path,
        language: str,
        *,
        start_date: Optional[str] = None,
        end_date: Optional[str] = None,
        difficulties: Optional[Sequence[str]] = None,
        holdout: Optional[str] = None,
        test_frac: float = 0.3,
        seed: int = 12345,
        shuffle: bool = False,
        max_instances: Optional[int] = None,
    ) -> "MultilingualLCBDataset":
        """Load stdin problems from a snapshot JSONL for ``language`` (testtypes forced)."""
        resolve_language(language)
        base = LiveCodeBenchDataset.from_jsonl(
            path,
            start_date=start_date,
            end_date=end_date,
            difficulties=difficulties,
            testtypes=_STDIN_ONLY,
            holdout=holdout,
            test_frac=test_frac,
            seed=seed,
            shuffle=shuffle,
            max_instances=max_instances,
        )
        return cls(base._rows, language)

to_jsonl(path)

Write the stdin rows as a language-independent snapshot (language is a per-run tag, not stored; reload with from_jsonl(path, language=...)).

Source code in genlm/eval/domains/livecodebench_multilingual/dataset.py

def to_jsonl(self, path) -> None:
    """Write the stdin rows as a language-independent snapshot (language is a per-run tag,
    not stored; reload with ``from_jsonl(path, language=...)``)."""
    LiveCodeBenchDataset(self._rows).to_jsonl(path)

from_hf(language, *, release='release_v6', start_date='2024-01-01', end_date=None, difficulties=None, holdout=None, test_frac=0.3, seed=12345, shuffle=False, max_instances=None, max_tests_per_problem=None, cumulative=True, cache_dir=None) classmethod

Load stdin LiveCodeBench problems for language (testtypes forced to stdin).

start_date defaults to the base loader's 2024-01-01; pass the paper's window (e.g. None or '2024-07-01') to match a specific problem set.

Source code in genlm/eval/domains/livecodebench_multilingual/dataset.py

@classmethod
def from_hf(
    cls,
    language: str,
    *,
    release: str = "release_v6",
    start_date: Optional[str] = "2024-01-01",
    end_date: Optional[str] = None,
    difficulties: Optional[Sequence[str]] = None,
    holdout: Optional[str] = None,
    test_frac: float = 0.3,
    seed: int = 12345,
    shuffle: bool = False,
    max_instances: Optional[int] = None,
    max_tests_per_problem: Optional[int] = None,
    cumulative: bool = True,
    cache_dir: Optional[str] = None,
) -> "MultilingualLCBDataset":
    """Load stdin LiveCodeBench problems for ``language`` (testtypes forced to stdin).

    ``start_date`` defaults to the base loader's ``2024-01-01``; pass the paper's window
    (e.g. ``None`` or ``'2024-07-01'``) to match a specific problem set.
    """
    resolve_language(language)  # validate early
    base = LiveCodeBenchDataset.from_hf(
        release=release,
        start_date=start_date,
        end_date=end_date,
        difficulties=difficulties,
        testtypes=_STDIN_ONLY,
        holdout=holdout,
        test_frac=test_frac,
        seed=seed,
        shuffle=shuffle,
        max_instances=max_instances,
        max_tests_per_problem=max_tests_per_problem,
        cumulative=cumulative,
        cache_dir=cache_dir,
    )
    return cls(base._rows, language)

from_jsonl(path, language, *, start_date=None, end_date=None, difficulties=None, holdout=None, test_frac=0.3, seed=12345, shuffle=False, max_instances=None) classmethod

Load stdin problems from a snapshot JSONL for language (testtypes forced).

Source code in genlm/eval/domains/livecodebench_multilingual/dataset.py

@classmethod
def from_jsonl(
    cls,
    path,
    language: str,
    *,
    start_date: Optional[str] = None,
    end_date: Optional[str] = None,
    difficulties: Optional[Sequence[str]] = None,
    holdout: Optional[str] = None,
    test_frac: float = 0.3,
    seed: int = 12345,
    shuffle: bool = False,
    max_instances: Optional[int] = None,
) -> "MultilingualLCBDataset":
    """Load stdin problems from a snapshot JSONL for ``language`` (testtypes forced)."""
    resolve_language(language)
    base = LiveCodeBenchDataset.from_jsonl(
        path,
        start_date=start_date,
        end_date=end_date,
        difficulties=difficulties,
        testtypes=_STDIN_ONLY,
        holdout=holdout,
        test_frac=test_frac,
        seed=seed,
        shuffle=shuffle,
        max_instances=max_instances,
    )
    return cls(base._rows, language)

MultilingualLCBInstance

Bases: LiveCodeBenchInstance

One LiveCodeBench stdin problem paired with a target language.

instance_id is the composite <question_id>@<language> (so the runner caches each language separately); question_id keeps the raw id for grouping/metadata.

Source code in genlm/eval/domains/livecodebench_multilingual/dataset.py

class MultilingualLCBInstance(LiveCodeBenchInstance):
    """One LiveCodeBench stdin problem paired with a target language.

    ``instance_id`` is the composite ``<question_id>@<language>`` (so the runner caches each
    language separately); ``question_id`` keeps the raw id for grouping/metadata.
    """

    language: str
    question_id: str

resolve_language(name)

Resolve a language name (case-insensitive, with aliases) to a Language; raises ValueError.

Source code in genlm/eval/domains/livecodebench_multilingual/dataset.py

def resolve_language(name: str) -> "Language":
    """Resolve a language name (case-insensitive, with aliases) to a Language; raises ValueError."""
    key = name.strip().lower()
    key = _ALIASES.get(key, key)
    if key not in LANGUAGES:
        raise ValueError(
            f"unknown language {name!r}; known: {sorted(LANGUAGES)} (aliases: {sorted(_ALIASES)})"
        )
    return LANGUAGES[key]

capture_run(code, inputs, outputs, language, timeout=10.0, grading='exact', max_completion_seconds=1000000000.0)

Run every test of one solution and return (solved, per_test_records), no short-circuit.

Each record has test_idx, passed, output (untruncated stdout), error_message, error_code (rollouts convention), status, and time_s. solved is all(per-test passed). A per-completion wall cap (max_completion_seconds) records tests past the cap as not-passed, status "capped".

Source code in genlm/eval/domains/livecodebench_multilingual/capture.py

def capture_run(
    code: str,
    inputs: List[str],
    outputs: List[str],
    language: str,
    timeout: float = 10.0,
    grading: str = "exact",
    max_completion_seconds: float = 1e9,
) -> Tuple[bool, List[dict]]:
    """Run every test of one solution and return (solved, per_test_records), no short-circuit.

    Each record has test_idx, passed, output (untruncated stdout), error_message, error_code
    (rollouts convention), status, and time_s. `solved` is all(per-test passed). A per-completion
    wall cap (max_completion_seconds) records tests past the cap as not-passed, status "capped".
    """
    n = len(outputs)
    if not (code or "").strip():
        recs = [
            _rec(
                i, False, "", "Empty string instead of a program", Status.EmptyCode, 0.0
            )
            for i in range(n)
        ]
        return False, recs

    code = patch_prog(code, language)
    ext = eval_scripts[language][1]
    recs: List[dict] = []
    t_start = time.time()

    with tempfile.TemporaryDirectory() as tmp:
        path = Path(tmp, "Main" + ext)
        with open(path, "w", encoding="utf8") as f:
            f.write(code)
            f.flush()
        sconf = SubprocessConfig(plang=language, run_timeout=max(1, int(ceil(timeout))))
        sconf.set_cwd(tmp)

        if language in _COMPILED:
            exe = str(path.with_suffix(".exe"))
            run_args, build_err = _build(language, path, exe, sconf)
            if run_args is None:
                return False, [
                    _rec(i, False, "", build_err, Status.BuildFailed, 0.0)
                    for i in range(n)
                ]
        elif language in _INTERPRETED:
            run_args = [a.format(path=str(path)) for a in _INTERPRETED[language]]
        else:
            raise NotImplementedError(
                f"capture_run has no recipe for language {language!r}"
            )

        for idx, (inp, exp) in enumerate(zip(inputs, outputs)):
            if time.time() - t_start > max_completion_seconds:
                recs.append(
                    _rec(idx, False, "", "completion wall cap reached", "capped", 0.0)
                )
                continue
            t0 = time.time()
            res = run(
                run_args, input_data=inp, timeout_seconds=sconf.run_timeout, sconf=sconf
            )
            dt = time.time() - t0
            out = (res.stdout or "").strip()
            rs = get_run_status(res)
            if rs != Status.Done:
                recs.append(_rec(idx, False, out, res.stderr, rs, dt))
            else:
                passed = _verdict(out, inp if inp is not None else "", exp, grading)
                recs.append(
                    _rec(idx, passed, out, "" if passed else res.stderr, rs, dt)
                )

    solved = bool(recs) and all(r["passed"] for r in recs)
    return solved, recs

MBPPAgnosticDataset

Bases: Dataset[MultilingualLCBInstance]

Ag-MBPP-X problems for one target language, validated and deduplicated.

platform='mbpp-agnostic' and question_id='mbppx_<task_id>' keep instances distinguishable from LCB problems.

Source code in genlm/eval/domains/livecodebench_multilingual/mbpp_agnostic.py

class MBPPAgnosticDataset(Dataset[MultilingualLCBInstance]):
    """Ag-MBPP-X problems for one target language, validated and deduplicated.

    ``platform='mbpp-agnostic'`` and ``question_id='mbppx_<task_id>'`` keep instances
    distinguishable from LCB problems.
    """

    def __init__(self, rows: List[Mapping[str, Any]], language: str,
                 drop_counts: Optional[Dict[str, int]] = None):
        self.language = resolve_language(language).key
        self._rows = list(rows)
        self.drop_counts = dict(drop_counts or {})

    def __len__(self) -> int:
        return len(self._rows)

    def __iter__(self) -> Iterator[MultilingualLCBInstance]:
        for row in self._rows:
            qid = row["question_id"]
            yield MultilingualLCBInstance(
                instance_id=f"{qid}@{self.language}",
                question_id=qid,
                language=self.language,
                question_content=row["question_content"],
                starter_code="",
                difficulty="unknown",
                platform="mbpp-agnostic",
                testtype="stdin",
                contest_date="",
                eval_sample=row["eval_sample"],
                public_eval_sample=row["public_eval_sample"],
            )

    @property
    def schema(self) -> Type[MultilingualLCBInstance]:
        return MultilingualLCBInstance

    @classmethod
    def from_rows(cls, raw_rows: List[Mapping[str, Any]], language: str,
                  *, strict: bool = False) -> "MBPPAgnosticDataset":
        """Validate raw rows: schema, size and character checks, an injection-marker scan,
        contradictory-test detection, and dedup by task id and normalized description.
        Failing rows are dropped and counted in ``drop_counts`` (``strict=True`` raises)."""
        rows: List[Dict[str, Any]] = []
        drop_counts: Dict[str, int] = {}
        seen_ids: set = set()
        seen_norm: set = set()
        for raw in raw_rows:
            drops: List[str] = []
            task_id = raw.get("original_task_id")
            if not isinstance(task_id, (int, str)) or str(task_id).strip() == "":
                drops.append("task_id:missing")
            desc = _check_text("description", raw.get("description"), drops)
            in_fmt = _check_text("input_format", raw.get("input_format"), drops)
            out_fmt = _check_text("output_format", raw.get("output_format"), drops)
            tests = _check_tests(raw.get("tests"), drops) if not drops else None
            if not drops:
                qid = f"mbppx_{task_id}"
                norm = _norm(desc)
                if qid in seen_ids:
                    drops.append("dedup:task_id")
                elif norm in seen_norm:
                    drops.append("dedup:description")
                else:
                    seen_ids.add(qid)
                    seen_norm.add(norm)
            if drops:
                if strict:
                    raise ValueError(f"row {raw.get('original_task_id')!r} failed: {drops}")
                for d in drops:
                    drop_counts[d] = drop_counts.get(d, 0) + 1
                continue
            # eval_sample carries all tests, example first, so public_eval_sample is a
            # prefix and grading matches the official framework
            rows.append({
                "question_id": qid,
                "question_content": _statement(desc, in_fmt, out_fmt, tests[0]),
                "eval_sample": _io_blob(tests),
                "public_eval_sample": _io_blob(tests[:1]),
            })
        return cls(rows, language, drop_counts)

    @classmethod
    def from_hf(cls, language: str, *, config: str = "sanitized",
                revision: str = PINNED_REVISION, strict: bool = False,
                cache_dir: Optional[str] = None) -> "MBPPAgnosticDataset":
        """Load from HF at the PINNED revision (pass ``revision`` explicitly to move it)."""
        from datasets import load_dataset

        ds = load_dataset(HF_REPO, config, revision=revision, cache_dir=cache_dir)
        raw = list(ds[next(iter(ds.keys()))])
        return cls.from_rows(raw, language, strict=strict)

    def overlap_with(self, other: "Dataset") -> List[Tuple[str, str]]:
        """Normalized-description collisions against another dataset (contamination check).

        Returns [(our question_id, their question_id)]; expected empty against LCB.
        """
        theirs = {_norm(i.question_content): i.question_id for i in other}
        hits = []
        for row in self._rows:
            n = _norm(row["question_content"])
            if n in theirs:
                hits.append((row["question_id"], theirs[n]))
        return hits

from_rows(raw_rows, language, *, strict=False) classmethod

Validate raw rows: schema, size and character checks, an injection-marker scan, contradictory-test detection, and dedup by task id and normalized description. Failing rows are dropped and counted in drop_counts (strict=True raises).

Source code in genlm/eval/domains/livecodebench_multilingual/mbpp_agnostic.py

@classmethod
def from_rows(cls, raw_rows: List[Mapping[str, Any]], language: str,
              *, strict: bool = False) -> "MBPPAgnosticDataset":
    """Validate raw rows: schema, size and character checks, an injection-marker scan,
    contradictory-test detection, and dedup by task id and normalized description.
    Failing rows are dropped and counted in ``drop_counts`` (``strict=True`` raises)."""
    rows: List[Dict[str, Any]] = []
    drop_counts: Dict[str, int] = {}
    seen_ids: set = set()
    seen_norm: set = set()
    for raw in raw_rows:
        drops: List[str] = []
        task_id = raw.get("original_task_id")
        if not isinstance(task_id, (int, str)) or str(task_id).strip() == "":
            drops.append("task_id:missing")
        desc = _check_text("description", raw.get("description"), drops)
        in_fmt = _check_text("input_format", raw.get("input_format"), drops)
        out_fmt = _check_text("output_format", raw.get("output_format"), drops)
        tests = _check_tests(raw.get("tests"), drops) if not drops else None
        if not drops:
            qid = f"mbppx_{task_id}"
            norm = _norm(desc)
            if qid in seen_ids:
                drops.append("dedup:task_id")
            elif norm in seen_norm:
                drops.append("dedup:description")
            else:
                seen_ids.add(qid)
                seen_norm.add(norm)
        if drops:
            if strict:
                raise ValueError(f"row {raw.get('original_task_id')!r} failed: {drops}")
            for d in drops:
                drop_counts[d] = drop_counts.get(d, 0) + 1
            continue
        # eval_sample carries all tests, example first, so public_eval_sample is a
        # prefix and grading matches the official framework
        rows.append({
            "question_id": qid,
            "question_content": _statement(desc, in_fmt, out_fmt, tests[0]),
            "eval_sample": _io_blob(tests),
            "public_eval_sample": _io_blob(tests[:1]),
        })
    return cls(rows, language, drop_counts)

from_hf(language, *, config='sanitized', revision=PINNED_REVISION, strict=False, cache_dir=None) classmethod

Load from HF at the PINNED revision (pass revision explicitly to move it).

Source code in genlm/eval/domains/livecodebench_multilingual/mbpp_agnostic.py

@classmethod
def from_hf(cls, language: str, *, config: str = "sanitized",
            revision: str = PINNED_REVISION, strict: bool = False,
            cache_dir: Optional[str] = None) -> "MBPPAgnosticDataset":
    """Load from HF at the PINNED revision (pass ``revision`` explicitly to move it)."""
    from datasets import load_dataset

    ds = load_dataset(HF_REPO, config, revision=revision, cache_dir=cache_dir)
    raw = list(ds[next(iter(ds.keys()))])
    return cls.from_rows(raw, language, strict=strict)

overlap_with(other)

Normalized-description collisions against another dataset (contamination check).

Returns [(our question_id, their question_id)]; expected empty against LCB.

Source code in genlm/eval/domains/livecodebench_multilingual/mbpp_agnostic.py

def overlap_with(self, other: "Dataset") -> List[Tuple[str, str]]:
    """Normalized-description collisions against another dataset (contamination check).

    Returns [(our question_id, their question_id)]; expected empty against LCB.
    """
    theirs = {_norm(i.question_content): i.question_id for i in other}
    hits = []
    for row in self._rows:
        n = _norm(row["question_content"])
        if n in theirs:
            hits.append((row["question_id"], theirs[n]))
    return hits

LocalSubprocessExecutor

Grade candidates by compiling/running them locally via the vendored executor.

No container isolation: generated code runs as host subprocesses with only rlimit + process-group SIGKILL. Run on a dedicated/disposable node only.

Source code in genlm/eval/domains/livecodebench_multilingual/executor.py

class LocalSubprocessExecutor:
    """Grade candidates by compiling/running them locally via the vendored executor.

    No container isolation: generated code runs as host subprocesses with only rlimit +
    process-group SIGKILL. Run on a dedicated/disposable node only.
    """

    def __init__(self, grading: str = "lenient") -> None:
        if grading not in ("lenient", "exact"):
            raise ValueError("grading must be 'lenient' or 'exact'")
        # "exact" = Agnostics whole-output rstrip equality; "lenient" = Multi-LCB's per-line
        # float-tolerant comparator (default).
        self.exact_match = grading == "exact"
        self._prepared: set[str] = set()

    def prepare(self, language: str) -> None:
        if language in self._prepared:
            return
        if language not in _tp.eval_scripts:
            raise NotImplementedError(
                f"language {language!r} is not yet wired in the vendored executor "
                f"(eval_scripts has {sorted(_tp.eval_scripts)})"
            )
        if not is_toolchain_available(language):
            raise RuntimeError(
                f"toolchain for {language!r} not found on PATH "
                f"(need {_TOOLCHAIN.get(language)}); install it or skip this language"
            )
        # Do not `go clean -cache` here: a cold stdlib rebuild can exceed the 60s build timeout,
        # while a warm cache builds fast.
        if language == "julia":
            # Warm Julia's precompile cache once; a cold first run can exceed a per-test timeout.
            try:
                subprocess.run(
                    ["julia", "--startup-file=no", "-e", "1+1"],
                    capture_output=True,
                    timeout=600,
                )
            except (OSError, subprocess.TimeoutExpired):
                pass
        self._prepared.add(language)

    def run(
        self,
        code: str,
        inputs: List[str],
        outputs: List[str],
        language: str,
        timeout: float,
    ) -> Tuple[bool, Dict[str, Any]]:
        if language not in _tp.eval_scripts:
            raise NotImplementedError(
                f"language {language!r} is not yet wired in the vendored executor "
                f"(eval_scripts has {sorted(_tp.eval_scripts)})"
            )
        # max(1, ...): eval_plang_code passes this to subprocess.communicate(timeout=...); a 0
        # would make every test time out instantly (mirrors livecodebench/harness.py).
        scores, meta = _tp.eval_plang_code(
            code,
            list(inputs),
            list(outputs),
            language,
            max(1, int(ceil(timeout))),
            exact_match=self.exact_match,
        )
        # Solved iff every per-test score is positive (PASSED=1); a failure yields a short list
        # ending in a non-positive score (FAILED or EXECFAIL).
        solved = bool(scores) and all(s.value > 0 for s in scores)
        metadata = {
            "status": str(getattr(meta, "error", "ok")),
            "per_test": [s.name for s in scores],
            "n_tests": len(outputs),
        }
        return solved, metadata

MultilingualCodeExecutor

Bases: Protocol

Source code in genlm/eval/domains/livecodebench_multilingual/executor.py

class MultilingualCodeExecutor(Protocol):
    def prepare(self, language: str) -> None:
        """One-time per-language setup before grading a batch (idempotent)."""
        ...  # pragma: no cover

    def run(
        self,
        code: str,
        inputs: List[str],
        outputs: List[str],
        language: str,
        timeout: float,
    ) -> Tuple[bool, Dict[str, Any]]:
        """Return (solved, metadata) for one candidate against the stdin/stdout tests."""
        ...  # pragma: no cover

prepare(language)

One-time per-language setup before grading a batch (idempotent).

Source code in genlm/eval/domains/livecodebench_multilingual/executor.py

def prepare(self, language: str) -> None:
    """One-time per-language setup before grading a batch (idempotent)."""
    ...  # pragma: no cover

run(code, inputs, outputs, language, timeout)

Return (solved, metadata) for one candidate against the stdin/stdout tests.

Source code in genlm/eval/domains/livecodebench_multilingual/executor.py

def run(
    self,
    code: str,
    inputs: List[str],
    outputs: List[str],
    language: str,
    timeout: float,
) -> Tuple[bool, Dict[str, Any]]:
    """Return (solved, metadata) for one candidate against the stdin/stdout tests."""
    ...  # pragma: no cover

is_toolchain_available(language)

True if every binary language needs is on PATH (python is always available).

Source code in genlm/eval/domains/livecodebench_multilingual/executor.py

def is_toolchain_available(language: str) -> bool:
    """True if every binary ``language`` needs is on PATH (python is always available)."""
    probes = _TOOLCHAIN.get(language)
    if probes is None:
        return False
    return all(shutil.which(b) is not None for b in probes)

agnostics_chat_messages(instance)

Agnostics Ag-LCB-X eval prompt: one user message naming the target language.

Mirrors agnostics-framework make_prompt_from_lcbx_row (a "# Problem / # Task" block, no system message). Pair with grading="exact" for an Agnostics-parity run.

Source code in genlm/eval/domains/livecodebench_multilingual/prompts.py

def agnostics_chat_messages(instance) -> List[Dict[str, str]]:
    """Agnostics Ag-LCB-X eval prompt: one user message naming the target language.

    Mirrors agnostics-framework make_prompt_from_lcbx_row (a "# Problem / # Task" block, no
    system message). Pair with grading="exact" for an Agnostics-parity run.
    """
    lang = resolve_language(instance.language)
    user = (
        f"# Problem\n{instance.question_content}\n\n"
        "# Task\nProvide a full implementation of the specified program in a Markdown code "
        f"block.\nUse the following programming language: {lang.key}\n"
    )
    return [{"role": "user", "content": user}]

chat_messages(instance)

Chat messages for instance in its source's prompt style: Multi-LCB languages get the Multi-LCB prompt, Agnostics low-resource languages get the Agnostics prompt with the per-language nudge. Prefer this over the style-specific builders so each prompt matches its dataset.

Source code in genlm/eval/domains/livecodebench_multilingual/prompts.py

def chat_messages(instance) -> List[Dict[str, str]]:
    """Chat messages for ``instance`` in its source's prompt style: Multi-LCB languages get the
    Multi-LCB prompt, Agnostics low-resource languages get the Agnostics prompt with the
    per-language nudge. Prefer this over the style-specific builders so each prompt matches its
    dataset."""
    lang = resolve_language(instance.language)
    if lang.source == "agnostics":
        msgs = agnostics_chat_messages(instance)
        if lang.prompt_nudge:
            msgs = [{**msgs[0], "content": msgs[0]["content"] + "\n" + lang.prompt_nudge}]
        return msgs
    return multilingual_chat_messages(instance)

default_grading(language)

Grading comparator matching each prompt source: exact (Agnostics rstrip-equality) for the Agnostics low-resource languages, lenient (Multi-LCB per-line comparator) otherwise.

Source code in genlm/eval/domains/livecodebench_multilingual/prompts.py

def default_grading(language) -> str:
    """Grading comparator matching each prompt source: ``exact`` (Agnostics rstrip-equality) for the
    Agnostics low-resource languages, ``lenient`` (Multi-LCB per-line comparator) otherwise."""
    return "exact" if resolve_language(language).source == "agnostics" else "lenient"

extract_code(model_output)

First fenced code block, matching Multi-LCB's extractor.

Drops a leading span, takes the first ``` block, and strips the "YOUR CODE HERE" placeholder. The Python-only domain's extractor takes the last block and keeps the placeholder.

Source code in genlm/eval/domains/livecodebench_multilingual/prompts.py

def extract_code(model_output) -> str:
    """First fenced code block, matching Multi-LCB's extractor.

    Drops a leading </think> span, takes the first ``` block, and strips the "YOUR CODE HERE"
    placeholder. The Python-only domain's extractor takes the last block and keeps the placeholder.
    """
    if not model_output:
        return ""
    t = model_output.find("</think>")
    if t >= 0:
        model_output = model_output[t + 8 :].strip()
    m = _CODE_BLOCK_RE.search(model_output)
    if not m:
        return ""
    return _PLACEHOLDER_RE.sub("", m.group(3))

format_multilingual_prompt(tokenizer, instance, use_chat_format=False, enable_thinking=None)

Build the multilingual LCB prompt for instance and return token ids.

use_chat_format=True applies the tokenizer's chat template (instruct models); otherwise the system and user messages are concatenated as a raw completion string. Mirrors the existing default_prompt_formatter interface.

Source code in genlm/eval/domains/livecodebench_multilingual/prompts.py

def format_multilingual_prompt(
    tokenizer,
    instance,
    use_chat_format: bool = False,
    enable_thinking: bool | None = None,
) -> List[int]:
    """Build the multilingual LCB prompt for ``instance`` and return token ids.

    use_chat_format=True applies the tokenizer's chat template (instruct models); otherwise
    the system and user messages are concatenated as a raw completion string. Mirrors the
    existing ``default_prompt_formatter`` interface.
    """
    messages = multilingual_chat_messages(instance)
    if use_chat_format and tokenizer is not None:
        kw = {} if enable_thinking is None else {"enable_thinking": enable_thinking}
        text = tokenizer.apply_chat_template(
            messages, tokenize=False, add_generation_prompt=True, **kw
        )
        # The chat template already inserts the BOS; avoid a second one on re-encode.
        return tokenizer.encode(text, add_special_tokens=False)
    text = f"{messages[0]['content']}\n\n{messages[1]['content']}"
    return tokenizer.encode(text)

format_prompt(tokenizer, instance, use_chat_format=False, enable_thinking=None)

Source-correct token ids for instance (Multi-LCB or Agnostics prompt by language source).

The generation-side analogue of format_multilingual_prompt but style-selecting via chat_messages. enable_thinking=None omits the toggle for models without a thinking mode.

Source code in genlm/eval/domains/livecodebench_multilingual/prompts.py

def format_prompt(
    tokenizer,
    instance,
    use_chat_format: bool = False,
    enable_thinking: bool | None = None,
) -> List[int]:
    """Source-correct token ids for ``instance`` (Multi-LCB or Agnostics prompt by language source).

    The generation-side analogue of ``format_multilingual_prompt`` but style-selecting via
    ``chat_messages``. ``enable_thinking=None`` omits the toggle for models without a thinking mode.
    """
    messages = chat_messages(instance)
    if use_chat_format and tokenizer is not None:
        kw = {} if enable_thinking is None else {"enable_thinking": enable_thinking}
        text = tokenizer.apply_chat_template(
            messages, tokenize=False, add_generation_prompt=True, **kw
        )
        return tokenizer.encode(text, add_special_tokens=False)
    # raw-completion fallback: agnostics carries one (user) message, Multi-LCB two (system+user)
    text = "\n\n".join(m["content"] for m in messages)
    return tokenizer.encode(text)

multilingual_chat_messages(instance)

The [system, user] chat messages for instance (for chat/API model adapters).

Source code in genlm/eval/domains/livecodebench_multilingual/prompts.py

def multilingual_chat_messages(instance) -> List[Dict[str, str]]:
    """The [system, user] chat messages for ``instance`` (for chat/API model adapters)."""
    lang = resolve_language(instance.language)
    return [
        {"role": "system", "content": _system_message(lang)},
        {"role": "user", "content": _user_body(instance.question_content, lang)},
    ]