Skip to content

fetch

genlm.eval.domains.livecodebench.fetch

Load + decode LiveCodeBench code_generation_lite releases.

Downloads raw testN.jsonl via huggingface_hub — the datasets builder hits a pyarrow offset-overflow on the large private_test_cases column.

derive_testtype(metadata, tests)

functional iff a func_name is present (matches run_test's which_type), else fall back to the first test's recorded testtype, else stdin.

Source code in genlm/eval/domains/livecodebench/fetch.py 
28
29
30
31
32
33
34
35
def derive_testtype(metadata: Mapping[str, Any], tests: List[Mapping[str, Any]]) -> str:
    """functional iff a func_name is present (matches run_test's which_type),
    else fall back to the first test's recorded testtype, else stdin."""
    if metadata.get("func_name"):
        return "functional"
    if tests and tests[0].get("testtype"):
        return str(tests[0]["testtype"])
    return "stdin"

build_row(raw, release, max_tests=None)

Convert a raw HF row into a clean snapshot row with a harness-ready eval_sample ({"input_output": <json str>}).

Source code in genlm/eval/domains/livecodebench/fetch.py 
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
def build_row(raw: Mapping[str, Any], release: str, max_tests: Optional[int] = None) -> Dict[str, Any]:
    """Convert a raw HF row into a clean snapshot row with a harness-ready
    ``eval_sample`` (``{"input_output": <json str>}``)."""
    public = json.loads(raw["public_test_cases"])
    private = _decode_private(raw["private_test_cases"])
    all_tests = list(public) + list(private)
    metadata = json.loads(raw["metadata"]) if isinstance(raw.get("metadata"), str) else (raw.get("metadata") or {})

    inputs = [t["input"] for t in all_tests]
    outputs = [t["output"] for t in all_tests]
    if max_tests is not None:
        inputs, outputs = inputs[:max_tests], outputs[:max_tests]

    fn_name = metadata.get("func_name")
    eval_sample = {"input_output": json.dumps({
        "inputs": inputs, "outputs": outputs, "fn_name": fn_name,
    })}
    # Public tests, separated so verifier potentials use them without leaking the
    # private set. Public sort first, so this is a prefix of eval_sample.
    n_public = len(public) if max_tests is None else min(len(public), max_tests)
    public_eval_sample = {"input_output": json.dumps({
        "inputs": inputs[:n_public], "outputs": outputs[:n_public], "fn_name": fn_name,
    })}
    return {
        "question_id": raw.get("question_id"),
        "question_content": raw.get("question_content", ""),
        "starter_code": raw.get("starter_code", "") or "",
        "difficulty": raw.get("difficulty", "unknown"),
        "platform": raw.get("platform", "unknown"),
        "contest_date": raw.get("contest_date", ""),
        "testtype": derive_testtype(metadata, all_tests),
        "release": release,
        "eval_sample": eval_sample,
        "public_eval_sample": public_eval_sample,
        "metadata": metadata,
    }

iter_release_rows(release='release_v6', max_tests=None, cache_dir=None, cumulative=True, raw_filter=None)

Yield clean built rows for a release (needs HF cache).

cumulative=True (official version_tag semantics) loads test.jsonl..testN.jsonl de-duped by question_id (release_v6 == ~1055); cumulative=False loads only that window. Dedup keeps the first occurrence, so release = first-seen.

raw_filter is applied to the raw HF row BEFORE the (expensive) private-test decode, so callers can drop e.g. out-of-window rows cheaply.

Source code in genlm/eval/domains/livecodebench/fetch.py 
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
def iter_release_rows(release: str = "release_v6", max_tests: Optional[int] = None,
                      cache_dir: Optional[str] = None, cumulative: bool = True,
                      raw_filter: Optional[Callable[[Mapping[str, Any]], bool]] = None,
                      ) -> Iterator[Dict[str, Any]]:
    """Yield clean built rows for a release (needs HF cache).

    ``cumulative=True`` (official version_tag semantics) loads test.jsonl..testN.jsonl
    de-duped by question_id (release_v6 == ~1055); ``cumulative=False`` loads only that
    window. Dedup keeps the first occurrence, so ``release`` = first-seen.

    ``raw_filter`` is applied to the raw HF row BEFORE the (expensive) private-test
    decode, so callers can drop e.g. out-of-window rows cheaply."""
    from huggingface_hub import hf_hub_download  # lazy: only from_hf needs the extra

    n = _release_num(release)
    tags = [f"release_v{i}" for i in range(1, n + 1)] if cumulative else [release]
    seen = set()
    for tag in tags:
        path = hf_hub_download(repo_id=HF_REPO, filename=_release_filename(tag),
                               repo_type="dataset", cache_dir=cache_dir)
        with open(path) as fin:
            for line in fin:
                if not line.strip():
                    continue
                raw = json.loads(line)
                if raw.get("question_id") in seen:
                    continue
                if raw_filter is not None and not raw_filter(raw):
                    continue
                row = build_row(raw, release=tag, max_tests=max_tests)
                seen.add(row["question_id"])
                yield row