prompts

genlm.eval.domains.livecodebench.prompts

Prompt formatting + code extraction kept identical to the official lcb_runner so numbers are leaderboard-comparable (base models: style="genericbase" = the official few-shot protocol, paired with whole-output extraction). NB lcb_runner uses the Meta-Llama-3-8B-Instruct chat template for all Llama-3.x instruct models.

format_lcb_prompt(row, tokenizer=None, chat_template=False, style='generic', enable_thinking=None)

Prompt for an lcb_runner LMStyle: "generic" (LLaMa3, via chat template when chat_template=True), "codeqwen" (CodeQwenInstruct, raw <|im_*|> string), or "deepseek" (DeepSeekCodeInstruct, raw ### Instruction/Response string).

enable_thinking forwards to apply_chat_template (Qwen3-style reasoning toggle); left out of the call when None so non-reasoning templates are unaffected.

Source code in genlm/eval/domains/livecodebench/prompts.py

def format_lcb_prompt(row: Mapping[str, str], tokenizer=None,
                      chat_template: bool = False, style: str = "generic",
                      enable_thinking: bool | None = None) -> str:
    """Prompt for an lcb_runner LMStyle: "generic" (LLaMa3, via chat template when
    chat_template=True), "codeqwen" (CodeQwenInstruct, raw <|im_*|> string), or
    "deepseek" (DeepSeekCodeInstruct, raw ### Instruction/Response string).

    enable_thinking forwards to apply_chat_template (Qwen3-style reasoning toggle); left
    out of the call when None so non-reasoning templates are unaffected."""
    if style not in STYLES:
        raise ValueError(f"style must be one of {STYLES}; got {style!r}")
    qc, sc = row.get("question_content", ""), row.get("starter_code", "") or ""
    if style == "codeqwen":
        # official joins system + body with a blank line ("...<|im_start|>user\n\n...")
        return f"{SYSTEM_MESSAGE_CODEQWEN}\n\n{_codeqwen_body(qc, sc)}"
    if style == "deepseek":
        return f"{SYSTEM_MESSAGE_DEEPSEEK}\n\n{_deepseek_body(qc, sc)}"
    if style == "genericbase":
        return _genericbase_body(qc, sc)
    body = _user_body(qc, sc)
    if chat_template and tokenizer is not None:
        messages = [{"role": "system", "content": SYSTEM_MESSAGE},
                    {"role": "user", "content": body}]
        kw = {} if enable_thinking is None else {"enable_thinking": enable_thinking}
        return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, **kw)
    return f"{SYSTEM_MESSAGE}\n\n{body}"

extract_code(model_output, style='generic')

Code between the last two ``` fences (last block if 3+); "" if fewer than two. style="genericbase" = whole stripped output. Matches lcb_runner extract_code.

Source code in genlm/eval/domains/livecodebench/prompts.py

def extract_code(model_output: str, style: str = "generic") -> str:
    """Code between the last two ``` fences (last block if 3+); "" if fewer than two.
    style="genericbase" = whole stripped output. Matches lcb_runner extract_code."""
    # Reasoning models (Qwen3, R1, ...) emit <think>...</think> before the answer; keep only the
    # post-think answer so a code fence inside the reasoning can't be mistaken for the solution.
    # No </think> (every existing non-reasoning model) leaves the output unchanged.
    if "</think>" in model_output:
        model_output = model_output.rsplit("</think>", 1)[1]
    if style == "genericbase":
        return model_output.strip()
    lines = model_output.split("\n")
    fence_idxs = [i for i, ln in enumerate(lines) if "```" in ln]
    if len(fence_idxs) < 2:
        return ""
    return "\n".join(lines[fence_idxs[-2] + 1: fence_idxs[-1]])

extract_code_prefix(model_output, style='generic')

Code being written, for prefix scoring: text after the last open fence, or "" when no block is open. Deferring on a closed block (a later block could supersede it) keeps prefix consistent with extract_code at complete. style="genericbase" = whole stripped output.

Source code in genlm/eval/domains/livecodebench/prompts.py

def extract_code_prefix(model_output: str, style: str = "generic") -> str:
    """Code being written, for prefix scoring: text after the last open fence, or
    "" when no block is open. Deferring on a closed block (a later block could
    supersede it) keeps prefix consistent with ``extract_code`` at complete.
    style="genericbase" = whole stripped output."""
    if style == "genericbase":
        return model_output.strip()
    lines = model_output.split("\n")
    fence_idxs = [i for i, ln in enumerate(lines) if "```" in ln]
    if len(fence_idxs) % 2 == 1:  # block open: judge text after the last fence
        return "\n".join(lines[fence_idxs[-1] + 1:])
    return ""  # closed or no block: defer to complete()

decode_context(context)

Decode a genlm.control context (str/bytes/list of byte tokens or int byte ids) into text.

Source code in genlm/eval/domains/livecodebench/prompts.py

def decode_context(context) -> str:
    """Decode a genlm.control context (str/bytes/list of byte tokens or int byte
    ids) into text."""
    if not context:
        return ""
    if isinstance(context, str):
        return context
    if isinstance(context, bytes):
        return context.decode("utf-8", errors="ignore")
    pieces = []
    for tok in context:
        if isinstance(tok, int):
            pieces.append(bytes([tok]))
        elif isinstance(tok, bytes):
            pieces.append(tok)
        else:
            pieces.append(str(tok).encode("utf-8", errors="ignore"))
    return b"".join(pieces).decode("utf-8", errors="ignore")