mbpp_agnostic

genlm.eval.domains.livecodebench_multilingual.mbpp_agnostic

Ag-MBPP-X (nuprl/mbpp-agnostic-translation) as a multilingual stdin/stdout eval set.

MBPP tasks rewritten by the Agnostics group into stdin/stdout form: an out-of-domain companion to the LCB problems, with no Codeforces overlap. Rows were reformulated from MBPP by Qwen3-32B (paper appendix), including the test I/O values, with no execution verification, so the loader pins the HF revision and validates every row; see from_rows() for the checks and drop accounting. Prompt construction and grading membership follow the official framework (tests[0] is the in-prompt example; eval_sample grades all tests, example first).

MBPPAgnosticDataset

Bases: Dataset[MultilingualLCBInstance]

Ag-MBPP-X problems for one target language, validated and deduplicated.

platform='mbpp-agnostic' and question_id='mbppx_<task_id>' keep instances distinguishable from LCB problems.

Source code in genlm/eval/domains/livecodebench_multilingual/mbpp_agnostic.py

class MBPPAgnosticDataset(Dataset[MultilingualLCBInstance]):
    """Ag-MBPP-X problems for one target language, validated and deduplicated.

    ``platform='mbpp-agnostic'`` and ``question_id='mbppx_<task_id>'`` keep instances
    distinguishable from LCB problems.
    """

    def __init__(self, rows: List[Mapping[str, Any]], language: str,
                 drop_counts: Optional[Dict[str, int]] = None):
        self.language = resolve_language(language).key
        self._rows = list(rows)
        self.drop_counts = dict(drop_counts or {})

    def __len__(self) -> int:
        return len(self._rows)

    def __iter__(self) -> Iterator[MultilingualLCBInstance]:
        for row in self._rows:
            qid = row["question_id"]
            yield MultilingualLCBInstance(
                instance_id=f"{qid}@{self.language}",
                question_id=qid,
                language=self.language,
                question_content=row["question_content"],
                starter_code="",
                difficulty="unknown",
                platform="mbpp-agnostic",
                testtype="stdin",
                contest_date="",
                eval_sample=row["eval_sample"],
                public_eval_sample=row["public_eval_sample"],
            )

    @property
    def schema(self) -> Type[MultilingualLCBInstance]:
        return MultilingualLCBInstance

    @classmethod
    def from_rows(cls, raw_rows: List[Mapping[str, Any]], language: str,
                  *, strict: bool = False) -> "MBPPAgnosticDataset":
        """Validate raw rows: schema, size and character checks, an injection-marker scan,
        contradictory-test detection, and dedup by task id and normalized description.
        Failing rows are dropped and counted in ``drop_counts`` (``strict=True`` raises)."""
        rows: List[Dict[str, Any]] = []
        drop_counts: Dict[str, int] = {}
        seen_ids: set = set()
        seen_norm: set = set()
        for raw in raw_rows:
            drops: List[str] = []
            task_id = raw.get("original_task_id")
            if not isinstance(task_id, (int, str)) or str(task_id).strip() == "":
                drops.append("task_id:missing")
            desc = _check_text("description", raw.get("description"), drops)
            in_fmt = _check_text("input_format", raw.get("input_format"), drops)
            out_fmt = _check_text("output_format", raw.get("output_format"), drops)
            tests = _check_tests(raw.get("tests"), drops) if not drops else None
            if not drops:
                qid = f"mbppx_{task_id}"
                norm = _norm(desc)
                if qid in seen_ids:
                    drops.append("dedup:task_id")
                elif norm in seen_norm:
                    drops.append("dedup:description")
                else:
                    seen_ids.add(qid)
                    seen_norm.add(norm)
            if drops:
                if strict:
                    raise ValueError(f"row {raw.get('original_task_id')!r} failed: {drops}")
                for d in drops:
                    drop_counts[d] = drop_counts.get(d, 0) + 1
                continue
            # eval_sample carries all tests, example first, so public_eval_sample is a
            # prefix and grading matches the official framework
            rows.append({
                "question_id": qid,
                "question_content": _statement(desc, in_fmt, out_fmt, tests[0]),
                "eval_sample": _io_blob(tests),
                "public_eval_sample": _io_blob(tests[:1]),
            })
        return cls(rows, language, drop_counts)

    @classmethod
    def from_hf(cls, language: str, *, config: str = "sanitized",
                revision: str = PINNED_REVISION, strict: bool = False,
                cache_dir: Optional[str] = None) -> "MBPPAgnosticDataset":
        """Load from HF at the PINNED revision (pass ``revision`` explicitly to move it)."""
        from datasets import load_dataset

        ds = load_dataset(HF_REPO, config, revision=revision, cache_dir=cache_dir)
        raw = list(ds[next(iter(ds.keys()))])
        return cls.from_rows(raw, language, strict=strict)

    def overlap_with(self, other: "Dataset") -> List[Tuple[str, str]]:
        """Normalized-description collisions against another dataset (contamination check).

        Returns [(our question_id, their question_id)]; expected empty against LCB.
        """
        theirs = {_norm(i.question_content): i.question_id for i in other}
        hits = []
        for row in self._rows:
            n = _norm(row["question_content"])
            if n in theirs:
                hits.append((row["question_id"], theirs[n]))
        return hits

from_rows(raw_rows, language, *, strict=False) classmethod

Validate raw rows: schema, size and character checks, an injection-marker scan, contradictory-test detection, and dedup by task id and normalized description. Failing rows are dropped and counted in drop_counts (strict=True raises).

Source code in genlm/eval/domains/livecodebench_multilingual/mbpp_agnostic.py

@classmethod
def from_rows(cls, raw_rows: List[Mapping[str, Any]], language: str,
              *, strict: bool = False) -> "MBPPAgnosticDataset":
    """Validate raw rows: schema, size and character checks, an injection-marker scan,
    contradictory-test detection, and dedup by task id and normalized description.
    Failing rows are dropped and counted in ``drop_counts`` (``strict=True`` raises)."""
    rows: List[Dict[str, Any]] = []
    drop_counts: Dict[str, int] = {}
    seen_ids: set = set()
    seen_norm: set = set()
    for raw in raw_rows:
        drops: List[str] = []
        task_id = raw.get("original_task_id")
        if not isinstance(task_id, (int, str)) or str(task_id).strip() == "":
            drops.append("task_id:missing")
        desc = _check_text("description", raw.get("description"), drops)
        in_fmt = _check_text("input_format", raw.get("input_format"), drops)
        out_fmt = _check_text("output_format", raw.get("output_format"), drops)
        tests = _check_tests(raw.get("tests"), drops) if not drops else None
        if not drops:
            qid = f"mbppx_{task_id}"
            norm = _norm(desc)
            if qid in seen_ids:
                drops.append("dedup:task_id")
            elif norm in seen_norm:
                drops.append("dedup:description")
            else:
                seen_ids.add(qid)
                seen_norm.add(norm)
        if drops:
            if strict:
                raise ValueError(f"row {raw.get('original_task_id')!r} failed: {drops}")
            for d in drops:
                drop_counts[d] = drop_counts.get(d, 0) + 1
            continue
        # eval_sample carries all tests, example first, so public_eval_sample is a
        # prefix and grading matches the official framework
        rows.append({
            "question_id": qid,
            "question_content": _statement(desc, in_fmt, out_fmt, tests[0]),
            "eval_sample": _io_blob(tests),
            "public_eval_sample": _io_blob(tests[:1]),
        })
    return cls(rows, language, drop_counts)

from_hf(language, *, config='sanitized', revision=PINNED_REVISION, strict=False, cache_dir=None) classmethod

Load from HF at the PINNED revision (pass revision explicitly to move it).

Source code in genlm/eval/domains/livecodebench_multilingual/mbpp_agnostic.py

@classmethod
def from_hf(cls, language: str, *, config: str = "sanitized",
            revision: str = PINNED_REVISION, strict: bool = False,
            cache_dir: Optional[str] = None) -> "MBPPAgnosticDataset":
    """Load from HF at the PINNED revision (pass ``revision`` explicitly to move it)."""
    from datasets import load_dataset

    ds = load_dataset(HF_REPO, config, revision=revision, cache_dir=cache_dir)
    raw = list(ds[next(iter(ds.keys()))])
    return cls.from_rows(raw, language, strict=strict)

overlap_with(other)

Normalized-description collisions against another dataset (contamination check).

Returns [(our question_id, their question_id)]; expected empty against LCB.

Source code in genlm/eval/domains/livecodebench_multilingual/mbpp_agnostic.py

def overlap_with(self, other: "Dataset") -> List[Tuple[str, str]]:
    """Normalized-description collisions against another dataset (contamination check).

    Returns [(our question_id, their question_id)]; expected empty against LCB.
    """
    theirs = {_norm(i.question_content): i.question_id for i in other}
    hits = []
    for row in self._rows:
        n = _norm(row["question_content"])
        if n in theirs:
            hits.append((row["question_id"], theirs[n]))
    return hits