testing_plang

genlm.eval.domains.livecodebench_multilingual.vendored.testing_plang

Vendored from Multi-LCB (MIT): the multilingual stdin/stdout code executor.

github.com/Multi-LCB/Multi-LCB @ d80be9f

lcb_runner/evaluation/testing_plang.py (blob 208624d)

Entry point: eval_plang_code(program, input_data, output_data, plang, timeout).

This copy is edited for genlm-eval (not verbatim); each change is marked with a "genlm-eval edit:" comment.

SubprocessConfig dataclass

TODO: maybe link with yaml files.

Source code in genlm/eval/domains/livecodebench_multilingual/vendored/testing_plang.py

@dataclass
class SubprocessConfig:
    """TODO: maybe link with yaml files."""

    plang: str  # plang or process name
    limit_memory: bool = True
    bufsize: Optional[int] = None
    env: dict = (
        None  # env variables, will overwrite current ENV variables for the process
    )
    build_timeout: int = 60
    run_timeout: int = 15  # time out for single test
    project_dir: Optional[str] = (
        None  # Sets the current directory before the subprocess is executed.
    )

    def __post_init__(self):
        plang = self.plang

        if self.bufsize is None:
            self.bufsize = DEFAULT_IO_BUF_SZ

        if self.env and "PATH" not in self.env:
            # all necessary variables must be copied from parent env.
            self.env["PATH"] = os.environ["PATH"]

        # genlm-eval edit: added "julia" - the JVM-free Julia runtime reserves a very large
        # virtual address space and hangs/times out under the 16GB RLIMIT_AS, like rust/js/ts.
        if plang in ("js", "ts", "javascript", "typescript", "rust", "julia"):
            self.limit_memory = False

    def set_cwd(self, cwd: str | Path):
        self.project_dir = Path(cwd)

run(args, timeout_seconds=8, input_data=None, sconf=None)

Runs the given program with arguments. After the timeout elapses, kills the process and all other processes in the process group. Captures at most max_output_size bytes of stdout and stderr each, and discards any output beyond that.

Source code in genlm/eval/domains/livecodebench_multilingual/vendored/testing_plang.py

def run(
    args: List[str],
    timeout_seconds: int = 8,
    input_data: str = None,
    sconf: SubprocessConfig = None,
) -> Result:
    """
    Runs the given program with arguments. After the timeout elapses, kills the process
    and all other processes in the process group. Captures at most max_output_size bytes
    of stdout and stderr each, and discards any output beyond that.
    """

    if not sconf:
        raise NotImplementedError("Subprocess config is expected")

    plang = sconf.plang
    env = sconf.env
    limit_memory = sconf.limit_memory
    bufsize = sconf.bufsize
    cwd = sconf.project_dir

    p = None  # global var sthat stores currenlty running subprocess

    # convert input data to stdin
    if input_data is not None:
        if input_data[-1:] != "\n":
            input_data += "\n"

        input_data = input_data.encode("utf-8")

    def _start_proc() -> Result:
        nonlocal p

        stdout, stderr = b"", b""
        exit_code = None

        p = subprocess.Popen(
            args,
            env=env,
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            start_new_session=True,
            # increase bufsize to fit estimated output size
            bufsize=4 * bufsize,
            cwd=cwd,
            preexec_fn=limit_virtual_memory if limit_memory else None,
        )

        # set_nonblocking(p.stdin)
        set_nonblocking(p.stdout)
        set_nonblocking(p.stderr)

        time.sleep(TIK)

        try:
            stdout, stderr = p.communicate(input=input_data, timeout=timeout_seconds)
            p.send_signal(signal.SIGINT)
            exit_code = p.returncode
        except subprocess.TimeoutExpired as ex:
            stderr = f"[{type(ex)}][{ex}]".encode("utf8")
        except Exception as ex:
            stderr = f"[{type(ex)}][{ex}]".encode("utf8")

        try:
            stdout = stdout.decode("utf-8")
            stderr = stderr.decode("utf-8")
        except Exception as ex:
            stdout = ""
            stderr = f"[{type(ex)}][{ex}]"
            # args = [str(s) for s in args]
            # print(f"ERROR! Can't decode stdout/stderr {stderr} on run('" + "', '".join(args) + "')")

        if len(stdout) > 64 * bufsize:
            # genlm-eval edit: replaced print() with logging so eval runs stay quiet.
            logger.warning(
                "output size (%d) exceeded expected (%d) by 64x; truncating",
                len(stdout),
                bufsize // 4,
            )
            stdout = stdout[: 64 * bufsize]

        return Result(plang=plang, exit_code=exit_code, stdout=stdout, stderr=stderr)

    try:
        result = _start_proc()
    finally:
        # Cleanup remaining zombie process
        kill_process(p)

    return result

check_js_runtime(program)

Check if script must be run using 'deno' runtime environment.

Examples: Deno.stdin Deno.readTextFromStdin Deno.readTextFileSync input = Deno.readAllSync(Deno.stdin) import { readline } from "https://deno.land/std@0.129.0/testing/readline.ts";

Source code in genlm/eval/domains/livecodebench_multilingual/vendored/testing_plang.py

def check_js_runtime(program) -> Literal["deno", "node"]:
    """Check if script must be run using 'deno' runtime environment.

    Examples:
    Deno.stdin
    Deno.readTextFromStdin
    Deno.readTextFileSync
    input = Deno.readAllSync(Deno.stdin)
    import { readline } from "https://deno.land/std@0.129.0/testing/readline.ts";

    """

    if re.search(
        "Deno[.]stdin|Deno[.]readTextFromStdin|Deno[.]readTextFileSync| Deno[.]|https://deno[.]land",
        program,
    ):
        return "deno"
    return "node"

install_npm_packages(sconf)

Install standard npm packages.

Source code in genlm/eval/domains/livecodebench_multilingual/vendored/testing_plang.py

def install_npm_packages(sconf: SubprocessConfig) -> Result:
    """Install standard npm packages."""

    # must be run without limits on memomry
    assert not sconf.limit_memory
    return run(
        [
            "npm",
            "i",
            "-D",
            "@types/node",
            "@types/readline-sync",
            "readline-sync",
            "yargs",
            "js-combinatorics",
        ],
        timeout_seconds=sconf.build_timeout,
        sconf=sconf,
    )

eval_script_php(path, input_data, sconf, **kwargs)

Evaluates a PHP script.

:param path: Path to the PHP source file. :type path: Path :param input_data: List of input strings for each test case. :type input_data: List[str] :param timeout_seconds: Timeout for execution in seconds. :type timeout_seconds: int :param bufsizes: List of buffer sizes for each test case. :type bufsizes: List[int | None] :param kwargs: Additional keyword arguments, including 'output_data'. :type kwargs: dict :return: A dictionary containing the status, exit code, stdout, and stderr. :rtype: dict

Source code in genlm/eval/domains/livecodebench_multilingual/vendored/testing_plang.py

def eval_script_php(
    path: Path,
    input_data: List[str],
    sconf: SubprocessConfig,
    **kwargs,
) -> dict:
    """
    Evaluates a PHP script.

    :param path: Path to the PHP source file.
    :type path: Path
    :param input_data: List of input strings for each test case.
    :type input_data: List[str]
    :param timeout_seconds: Timeout for execution in seconds.
    :type timeout_seconds: int
    :param bufsizes: List of buffer sizes for each test case.
    :type bufsizes: List[int | None]
    :param kwargs: Additional keyword arguments, including 'output_data'.
    :type kwargs: dict
    :return: A dictionary containing the status, exit code, stdout, and stderr.
    :rtype: dict
    """
    outputs, errors, status = [], [], None

    if not input_data:
        input_data = [None]

    # Original loop for multiple test cases with input_data
    for input_str in input_data:
        result = run(
            ["php", str(path)],
            input_data=input_str,
            timeout_seconds=sconf.run_timeout,
            sconf=sconf,
        )
        outputs.append(result.stdout)
        errors.append(result.stderr)
        status = get_run_status(result)

        if status.is_failure():
            break

    if status is None and result.exit_code == 0:
        status = Status.Done

    return {
        "status": status,
        "exit_code": result.exit_code,
        "stdout": outputs,
        "stderr": errors,
    }

eval_script_scala(path, input_data, sconf, **kwargs)

Evaluates a Scala script by compiling directly with scalac and running with scala.

Source code in genlm/eval/domains/livecodebench_multilingual/vendored/testing_plang.py

def eval_script_scala(
    path: Path,
    input_data: List[str],
    sconf: SubprocessConfig,
    **kwargs,
) -> dict:
    """
    Evaluates a Scala script by compiling directly with scalac and running with scala.
    """

    outputs, errors, status = [], [], None
    project_dir = sconf.project_dir

    # Extract class name from file
    class_name = _extract_scala_object_name(kwargs["code"])

    # Compile with scalac
    result = run(
        ["scalac", "-d", project_dir, str(path)],
        timeout_seconds=sconf.build_timeout,
        sconf=sconf,
    )

    status = get_build_status(result)

    if status != Status.BuildDone:
        outputs.append(result.stdout)
        errors.append(result.stderr)
    else:
        for input_str in input_data:
            # Run with scala
            result = run(
                ["scala", "-cp", project_dir, class_name],
                input_data=input_str,
                timeout_seconds=sconf.run_timeout,
                sconf=sconf,
            )

            outputs.append(result.stdout)
            errors.append(result.stderr)

            status = get_run_status(result)
            if status.is_failure():
                break

    # Clean up compiled class files
    for class_file in project_dir.glob("*.class"):
        class_file.unlink()

    if status is None and result.exit_code == 0:
        status = Status.Done

    return {
        "status": status,
        "exit_code": result.exit_code,
        "stdout": outputs,
        "stderr": errors,
    }

eval_script_kotlin(path, input_data, sconf, **kwargs)

Evaluates a Kotlin script.

:param path: Path to the Kotlin source file. :type path: Path :param input_data: List of input strings for each test case. :type input_data: List[str] :param timeout_seconds: Timeout for execution in seconds. :type timeout_seconds: int :param bufsizes: List of buffer sizes for each test case. :type bufsizes: List[int | None] :param kwargs: Additional keyword arguments, including 'program' and 'output_data'. :type kwargs: dict :raises RuntimeError: If there is an issue during Kotlin compilation or execution. :return: A dictionary containing the status, exit code, stdout, and stderr. :rtype: dict

Source code in genlm/eval/domains/livecodebench_multilingual/vendored/testing_plang.py

def eval_script_kotlin(
    path: Path,
    input_data: List[str],
    sconf: SubprocessConfig,
    **kwargs,
) -> dict:
    """
    Evaluates a Kotlin script.

    :param path: Path to the Kotlin source file.
    :type path: Path
    :param input_data: List of input strings for each test case.
    :type input_data: List[str]
    :param timeout_seconds: Timeout for execution in seconds.
    :type timeout_seconds: int
    :param bufsizes: List of buffer sizes for each test case.
    :type bufsizes: List[int | None]
    :param kwargs: Additional keyword arguments, including 'program' and 'output_data'.
    :type kwargs: dict
    :raises RuntimeError: If there is an issue during Kotlin compilation or execution.
    :return: A dictionary containing the status, exit code, stdout, and stderr.
    :rtype: dict
    """
    outputs, errors, status = [], [], None
    exec_name = path.with_suffix(".jar")

    result = run(
        ["kotlinc", str(path), "-include-runtime", "-d", str(exec_name)],
        timeout_seconds=sconf.build_timeout,
        sconf=sconf,
    )

    if result.exit_code is None:
        status = Status.BuildFailed
        outputs.append(result.stdout)
        errors.append(result.stderr)
    elif result.exit_code != 0:
        status = Status.SyntaxError
        outputs.append(result.stdout)
        errors.append(result.stderr)
    else:
        for input_str in input_data:
            result = run(
                ["java", "-jar", str(exec_name)],
                input_data=input_str,
                sconf=sconf,
                timeout_seconds=sconf.run_timeout,
            )

            outputs.append(result.stdout)
            errors.append(result.stderr)

            status = get_run_status(result)
            if status.is_failure():
                break

    if status is None and result.exit_code == 0:
        status = Status.Done

    return {
        "status": status,
        "exit_code": result.exit_code,
        "stdout": outputs,
        "stderr": errors,
    }

patch_prog(program, plang)

Minor compilation/run errors can be fixed by patching the code.

Parameters:

Name	Type	Description	Default
program	str	code of a program	required
plang	str	name of a programming language	required

Returns:

Type	Description
str	patched program code

Source code in genlm/eval/domains/livecodebench_multilingual/vendored/testing_plang.py

def patch_prog(program: str, plang: str) -> str:
    """Minor compilation/run errors can be fixed by patching the code.

    Args:
        program (str): code of a program
        plang (str): name of a programming language

    Returns:
        (str): patched program code
    """
    plang = plang.lower()
    if plang == "python":
        # patch blas: fix numpy import error when working in single thread
        patch = "import os\nos.environ['OPENBLAS_NUM_THREADS'] = '1'\n"
        program = patch + program
    elif plang == "c++":
        pass
    elif plang == "go":
        pass
    elif plang == "javascript":
        pass

    return program

eval_plang_code(program, input_data, output_data, plang, timeout, exact_match=False)

Main entry point.

Parameters:

Name	Type	Description	Default
program	str	program code.	required
input_data	List[str]	tests input data in stdin format. Each string is one test.	required
output_data	List[str]	tests output data in stdin format. This list is matched correspondingly to input data.	required
plang	str	name of the programming language ["c++","c#", ...]	required
timeout	int	test timeout	required
exact_match	bool	genlm-eval edit - if True, grade with Agnostics-style whole-output rstrip equality (match_tests_exact) instead of the default lenient comparator.	False

Returns:

Name	Type	Description
EvalScores	EvalScores	list with scores for each test
ResultMeta	ResultMeta	information on error or other problems during execution

Source code in genlm/eval/domains/livecodebench_multilingual/vendored/testing_plang.py

def eval_plang_code(
    program: str,
    input_data: List[str],
    output_data: List[str],
    plang: str,
    timeout: int,
    exact_match: bool = False,
) -> Tuple[EvalScores, ResultMeta]:
    """Main entry point.

    Args:
        program (str): program code.
        input_data (List[str]): tests input data in stdin format. Each string is one test.
        output_data (List[str]): tests output data in stdin format. This list is matched correspondingly to input data.
        plang (str): name of the programming language ["c++","c#", ...]
        timeout (int): test timeout
        exact_match (bool): genlm-eval edit - if True, grade with Agnostics-style whole-output
            rstrip equality (match_tests_exact) instead of the default lenient comparator.

    Returns:
        EvalScores: list with scores for each test
        ResultMeta: information on error or other problems during execution
    """

    if program is None or program == "":
        return [TestScore.EXECFAIL], NoCodeMeta()

    start = time.time()

    program = patch_prog(program, plang)

    # Compile and run the program
    sconf = SubprocessConfig(plang=plang, run_timeout=timeout)
    res = compile_and_run(program, input_data, sconf=sconf)

    total_exec_time = time.time() - start

    if res["status"] != Status.Done:
        result = [TestScore.EXECFAIL]
        metadata = ExecutionErrorMeta(
            error=res["status"],
            error_code=res["exit_code"],
            error_message=res["stderr"],
        )
        return result, metadata

    if exact_match:
        all_results, metadata = match_tests_exact(res["stdout"], output_data)
    else:
        all_results, metadata = match_tests_groud_truth(
            res["stdout"], input_data, output_data
        )

    if not metadata.success:
        return all_results, metadata

    # update output with real execution time
    return all_results, SuccessRunMeta(execution_time=total_exec_time)

match_tests_groud_truth(code_outputs, input_data, output_data)

Compare code outputs with ground truth.

Parameters:

Name	Type	Description	Default
code_outputs	List[str]	results of code execution, stdout format, one variable for each test	required
input_data	List[str]	tests input data, stdin format, one variable for each test	required
output_data	List[str]	expected tests outputs, stdout format, one variable for each test	required

Returns:

Type	Description
Tuple[EvalScores, WrongAnswerMeta | SuccessRunMeta]	Tuple[EvalScores, WrongAnswerMeta | SuccessRunMeta]: description

Source code in genlm/eval/domains/livecodebench_multilingual/vendored/testing_plang.py

def match_tests_groud_truth(
    code_outputs: List[str], input_data: List[str], output_data: List[str]
) -> Tuple[EvalScores, WrongAnswerMeta | SuccessRunMeta]:
    """Compare code outputs with ground truth.

    Args:
        code_outputs (List[str]): results of code execution, stdout format, one variable for each test
        input_data (List[str]): tests input data, stdin format, one variable for each test
        output_data (List[str]): expected tests outputs, stdout format, one variable for each test

    Returns:
        Tuple[EvalScores, WrongAnswerMeta | SuccessRunMeta]: _description_
    """
    epsilon = 1e-5  # LeetCode format for floats precision

    ## Compare code output with expected output
    all_results = []
    for prediction, gt_inp, gt_out in zip_longest(
        code_outputs, input_data, output_data, fillvalue="None"
    ):

        stripped_prediction_lines = get_stripped_lines(prediction)
        stripped_gt_out_lines = get_stripped_lines(gt_out)

        ## WA happens in multiple circumstances
        ## so cache the return to make it clean!
        wa_meta = WrongAnswerMeta(
            output=truncatefn(prediction),
            inputs=truncatefn(gt_inp),
            expected=truncatefn(gt_out),
            error_message="",  # will be added later
        )

        if len(stripped_prediction_lines) != len(stripped_gt_out_lines):
            all_results.append(TestScore.FAILED)
            wa_meta.error_message = "Wrong answer: mismatched output length"
            return all_results, wa_meta

        for output_line_idx, (
            stripped_prediction_line,
            stripped_gt_out_line,
        ) in enumerate(zip(stripped_prediction_lines, stripped_gt_out_lines)):

            # prepare output message in case of WA
            wa_meta.error_message = f"Wrong answer at {output_line_idx=}: {truncatefn(stripped_prediction_line)} != {truncatefn(stripped_gt_out_line)}"

            ## CASE 1: exact match
            if stripped_prediction_line == stripped_gt_out_line:
                continue

            ## CASE 2: bool match
            if stripped_prediction_line in [
                "True",
                "true",
            ] and stripped_gt_out_line in ["True", "true"]:
                continue

            if stripped_prediction_line in [
                "False",
                "false",
            ] and stripped_gt_out_line in ["False", "false"]:
                continue

            ## CASE 3: element-wise comparision
            ## if there are floating elements
            ## note that we should always be able to convert to decimals

            success, decimal_prediction_line = convert_line_to_decimals(
                stripped_prediction_line
            )
            if not success:
                all_results.append(TestScore.FAILED)
                return all_results, wa_meta

            success, decimal_gtout_line = convert_line_to_decimals(stripped_gt_out_line)

            if not success:
                all_results.append(TestScore.FAILED)
                return all_results, wa_meta

            if len(decimal_prediction_line) == len(decimal_gtout_line):
                # check all Decimals are close

                all_good = all(
                    [
                        isclose(a, b, abs_tol=epsilon, rel_tol=0)
                        for a, b in zip(decimal_prediction_line, decimal_gtout_line)
                    ]
                )

                if all_good:
                    continue

            all_results.append(TestScore.FAILED)
            return all_results, wa_meta

        all_results.append(TestScore.PASSED)

    return all_results, SuccessRunMeta(execution_time=-1)

match_tests_exact(code_outputs, output_data)

genlm-eval edit: Agnostics-style grading, whole-output rstrip equality per test.

The agnostics-framework executors compare real_output.rstrip() != expected_output.rstrip() once per test (all must pass). This is stricter than match_tests_groud_truth: no per-line split, no True/False aliasing, no float tolerance. Used for Agnostics parity. code_outputs are already whitespace-stripped by compile_and_run, so a leading-whitespace difference (which agnostics would keep) is not distinguished here.

Source code in genlm/eval/domains/livecodebench_multilingual/vendored/testing_plang.py

def match_tests_exact(
    code_outputs: List[str], output_data: List[str]
) -> Tuple[EvalScores, "WrongAnswerMeta | SuccessRunMeta"]:
    """genlm-eval edit: Agnostics-style grading, whole-output rstrip equality per test.

    The agnostics-framework executors compare `real_output.rstrip() != expected_output.rstrip()`
    once per test (all must pass). This is stricter than match_tests_groud_truth: no per-line
    split, no True/False aliasing, no float tolerance. Used for Agnostics parity. code_outputs
    are already whitespace-stripped by compile_and_run, so a leading-whitespace difference
    (which agnostics would keep) is not distinguished here.
    """
    all_results = []
    for prediction, gt_out in zip_longest(code_outputs, output_data, fillvalue=None):
        if (
            prediction is None
            or gt_out is None
            or prediction.rstrip() != gt_out.rstrip()
        ):
            all_results.append(TestScore.FAILED)
            return all_results, WrongAnswerMeta(
                output=truncatefn(prediction),
                expected=truncatefn(gt_out),
                error_message="Wrong answer (exact-match)",
            )
        all_results.append(TestScore.PASSED)
    return all_results, SuccessRunMeta(execution_time=-1)

prepare_plang_env(plang, timeout=None)

Some languages might fail to compile and run if your system is not prepared properly.

Parameters:

Name	Type	Description	Default
plang	str	name of a programming language.	required
timeout	int	value of the timeout that will be passed to process. Defaults to None.	None

Returns:

Type	Description
None	None

Source code in genlm/eval/domains/livecodebench_multilingual/vendored/testing_plang.py

def prepare_plang_env(plang: str, timeout: int = None) -> None:
    """Some languages might fail to compile and run if your system is not prepared properly.

    Args:
        plang (str): name of a programming language.
        timeout (int, optional): value of the timeout that will be passed to process. Defaults to None.

    Returns:
        None
    """

    if plang == "go":
        # if go cache is full all go tasks may fail with Timeout
        # 5 min timeout will give enough cache space. Otherwise cache clean might take up to an hour.
        if not timeout:
            timeout = 60 * 5

        sconf = SubprocessConfig(plang="go", limit_memory=False, build_timeout=timeout)

        # genlm-eval edit: replaced print() with logging.
        logger.info("cleaning go build cache (go clean -cache)...")
        _ = run(
            ["go", "clean", "-cache"], timeout_seconds=sconf.build_timeout, sconf=sconf
        )

    return None