Skip to content

prompts

genlm.eval.domains.livecodebench_multilingual.prompts

Per-language prompt construction for multilingual LiveCodeBench (stdin/stdout problems).

Mirrors Multi-LCB's stdin prompt so the 12 Multi-LCB languages are prompt-comparable to the paper. extract_code matches Multi-LCB's extractor (first fenced block, placeholder stripped), not the Python-only domain's, so grading matches the Multi-LCB pipeline.

extract_code(model_output)

First fenced code block, matching Multi-LCB's extractor.

Drops a leading span, takes the first ``` block, and strips the "YOUR CODE HERE" placeholder. The Python-only domain's extractor takes the last block and keeps the placeholder.

Source code in genlm/eval/domains/livecodebench_multilingual/prompts.py 
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
def extract_code(model_output) -> str:
    """First fenced code block, matching Multi-LCB's extractor.

    Drops a leading </think> span, takes the first ``` block, and strips the "YOUR CODE HERE"
    placeholder. The Python-only domain's extractor takes the last block and keeps the placeholder.
    """
    if not model_output:
        return ""
    t = model_output.find("</think>")
    if t >= 0:
        model_output = model_output[t + 8 :].strip()
    m = _CODE_BLOCK_RE.search(model_output)
    if not m:
        return ""
    return _PLACEHOLDER_RE.sub("", m.group(3))

multilingual_chat_messages(instance)

The [system, user] chat messages for instance (for chat/API model adapters).

Source code in genlm/eval/domains/livecodebench_multilingual/prompts.py 
75
76
77
78
79
80
81
def multilingual_chat_messages(instance) -> List[Dict[str, str]]:
    """The [system, user] chat messages for ``instance`` (for chat/API model adapters)."""
    lang = resolve_language(instance.language)
    return [
        {"role": "system", "content": _system_message(lang)},
        {"role": "user", "content": _user_body(instance.question_content, lang)},
    ]

agnostics_chat_messages(instance)

Agnostics Ag-LCB-X eval prompt: one user message naming the target language.

Mirrors agnostics-framework make_prompt_from_lcbx_row (a "# Problem / # Task" block, no system message). Pair with grading="exact" for an Agnostics-parity run.

Source code in genlm/eval/domains/livecodebench_multilingual/prompts.py 
84
85
86
87
88
89
90
91
92
93
94
95
96
def agnostics_chat_messages(instance) -> List[Dict[str, str]]:
    """Agnostics Ag-LCB-X eval prompt: one user message naming the target language.

    Mirrors agnostics-framework make_prompt_from_lcbx_row (a "# Problem / # Task" block, no
    system message). Pair with grading="exact" for an Agnostics-parity run.
    """
    lang = resolve_language(instance.language)
    user = (
        f"# Problem\n{instance.question_content}\n\n"
        "# Task\nProvide a full implementation of the specified program in a Markdown code "
        f"block.\nUse the following programming language: {lang.key}\n"
    )
    return [{"role": "user", "content": user}]

format_multilingual_prompt(tokenizer, instance, use_chat_format=False, enable_thinking=None)

Build the multilingual LCB prompt for instance and return token ids.

use_chat_format=True applies the tokenizer's chat template (instruct models); otherwise the system and user messages are concatenated as a raw completion string. Mirrors the existing default_prompt_formatter interface.

Source code in genlm/eval/domains/livecodebench_multilingual/prompts.py 
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
def format_multilingual_prompt(
    tokenizer,
    instance,
    use_chat_format: bool = False,
    enable_thinking: bool | None = None,
) -> List[int]:
    """Build the multilingual LCB prompt for ``instance`` and return token ids.

    use_chat_format=True applies the tokenizer's chat template (instruct models); otherwise
    the system and user messages are concatenated as a raw completion string. Mirrors the
    existing ``default_prompt_formatter`` interface.
    """
    messages = multilingual_chat_messages(instance)
    if use_chat_format and tokenizer is not None:
        kw = {} if enable_thinking is None else {"enable_thinking": enable_thinking}
        text = tokenizer.apply_chat_template(
            messages, tokenize=False, add_generation_prompt=True, **kw
        )
        # The chat template already inserts the BOS; avoid a second one on re-encode.
        return tokenizer.encode(text, add_special_tokens=False)
    text = f"{messages[0]['content']}\n\n{messages[1]['content']}"
    return tokenizer.encode(text)

chat_messages(instance)

Chat messages for instance in its source's prompt style: Multi-LCB languages get the Multi-LCB prompt, Agnostics low-resource languages get the Agnostics prompt with the per-language nudge. Prefer this over the style-specific builders so each prompt matches its dataset.

Source code in genlm/eval/domains/livecodebench_multilingual/prompts.py 
123
124
125
126
127
128
129
130
131
132
133
134
def chat_messages(instance) -> List[Dict[str, str]]:
    """Chat messages for ``instance`` in its source's prompt style: Multi-LCB languages get the
    Multi-LCB prompt, Agnostics low-resource languages get the Agnostics prompt with the
    per-language nudge. Prefer this over the style-specific builders so each prompt matches its
    dataset."""
    lang = resolve_language(instance.language)
    if lang.source == "agnostics":
        msgs = agnostics_chat_messages(instance)
        if lang.prompt_nudge:
            msgs = [{**msgs[0], "content": msgs[0]["content"] + "\n" + lang.prompt_nudge}]
        return msgs
    return multilingual_chat_messages(instance)

default_grading(language)

Grading comparator matching each prompt source: exact (Agnostics rstrip-equality) for the Agnostics low-resource languages, lenient (Multi-LCB per-line comparator) otherwise.

Source code in genlm/eval/domains/livecodebench_multilingual/prompts.py 
137
138
139
140
def default_grading(language) -> str:
    """Grading comparator matching each prompt source: ``exact`` (Agnostics rstrip-equality) for the
    Agnostics low-resource languages, ``lenient`` (Multi-LCB per-line comparator) otherwise."""
    return "exact" if resolve_language(language).source == "agnostics" else "lenient"

format_prompt(tokenizer, instance, use_chat_format=False, enable_thinking=None)

Source-correct token ids for instance (Multi-LCB or Agnostics prompt by language source).

The generation-side analogue of format_multilingual_prompt but style-selecting via chat_messages. enable_thinking=None omits the toggle for models without a thinking mode.

Source code in genlm/eval/domains/livecodebench_multilingual/prompts.py 
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
def format_prompt(
    tokenizer,
    instance,
    use_chat_format: bool = False,
    enable_thinking: bool | None = None,
) -> List[int]:
    """Source-correct token ids for ``instance`` (Multi-LCB or Agnostics prompt by language source).

    The generation-side analogue of ``format_multilingual_prompt`` but style-selecting via
    ``chat_messages``. ``enable_thinking=None`` omits the toggle for models without a thinking mode.
    """
    messages = chat_messages(instance)
    if use_chat_format and tokenizer is not None:
        kw = {} if enable_thinking is None else {"enable_thinking": enable_thinking}
        text = tokenizer.apply_chat_template(
            messages, tokenize=False, add_generation_prompt=True, **kw
        )
        return tokenizer.encode(text, add_special_tokens=False)
    # raw-completion fallback: agnostics carries one (user) message, Multi-LCB two (system+user)
    text = "\n\n".join(m["content"] for m in messages)
    return tokenizer.encode(text)