From d0b5d30aacf9aff687c4259e0afcc0238240076c Mon Sep 17 00:00:00 2001 From: Jason Gross Date: Tue, 26 Dec 2023 21:38:42 -0800 Subject: [PATCH 1/3] Use the more portable `.` instead of `source` This avoids errors such as `/bin/sh: 1: source: not found` ``` $ python harness/run_evaluation.py --predictions_path example-predictions.json --swe_bench_tasks example.json --log_dir example-log/ --testbed example-testbed/ Error: Command 'source /home/jgross/Documents/GitHub/SWE-bench/example-testbed/fakemodel/sqlfluff__sqlfluff/1.4/tmpdx4urtao/miniconda3/bin/activate sqlfluff__sqlfluff__1.4 && echo 'activate successful' && pip install -r /home/jgross/Documents/GitHub/SWE-bench/example-testbed/fakemodel/sqlfluff__sqlfluff/1.4/tmpfnim3stm/requirements.txt' returned non-zero exit status 127. Error stdout: Error stderr: /bin/sh: 1: source: not found Error traceback: Traceback (most recent call last): File "/home/jgross/Documents/GitHub/SWE-bench/harness/context_manager.py", line 50, in __call__ output = subprocess.run(cmd, **combined_args) File "/home/jgross/.local64/mambaforge/envs/swe-bench/lib/python3.9/subprocess.py", line 528, in run raise CalledProcessError(retcode, process.args, subprocess.CalledProcessError: Command 'source /home/jgross/Documents/GitHub/SWE-bench/example-testbed/fakemodel/sqlfluff__sqlfluff/1.4/tmpdx4urtao/miniconda3/bin/activate sqlfluff__sqlfluff__1.4 && echo 'activate successful' && pip install -r /home/jgross/Documents/GitHub/SWE-bench/example-testbed/fakemodel/sqlfluff__sqlfluff/1.4/tmpfnim3stm/requirements.txt' returned non-zero exit status 127. Traceback (most recent call last): File "/home/jgross/Documents/GitHub/SWE-bench/harness/run_evaluation.py", line 186, in main(**vars(args)) File "/home/jgross/Documents/GitHub/SWE-bench/harness/run_evaluation.py", line 162, in main eval_engine(args) File "/home/jgross/Documents/GitHub/SWE-bench/harness/engine_evaluation.py", line 164, in main setup_testbed(data_groups[0]) File "/home/jgross/Documents/GitHub/SWE-bench/harness/engine_validation.py", line 89, in setup_testbed with TestbedContextManager( File "/home/jgross/Documents/GitHub/SWE-bench/harness/context_manager.py", line 288, in __enter__ self.exec(cmd, shell=True) File "/home/jgross/Documents/GitHub/SWE-bench/harness/context_manager.py", line 58, in __call__ raise e File "/home/jgross/Documents/GitHub/SWE-bench/harness/context_manager.py", line 50, in __call__ output = subprocess.run(cmd, **combined_args) File "/home/jgross/.local64/mambaforge/envs/swe-bench/lib/python3.9/subprocess.py", line 528, in run raise CalledProcessError(retcode, process.args, subprocess.CalledProcessError: Command 'source /home/jgross/Documents/GitHub/SWE-bench/example-testbed/fakemodel/sqlfluff__sqlfluff/1.4/tmpdx4urtao/miniconda3/bin/activate sqlfluff__sqlfluff__1.4 && echo 'activate successful' && pip install -r /home/jgross/Documents/GitHub/SWE-bench/example-testbed/fakemodel/sqlfluff__sqlfluff/1.4/tmpfnim3stm/requirements.txt' returned non-zero exit status 127. ``` --- collect/cleanup/remove_envs.py | 2 +- harness/context_manager.py | 9 +++------ versioning/get_versions.py | 4 ++-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/collect/cleanup/remove_envs.py b/collect/cleanup/remove_envs.py index 3fd130a6..d19ea8de 100644 --- a/collect/cleanup/remove_envs.py +++ b/collect/cleanup/remove_envs.py @@ -69,7 +69,7 @@ def remove_environment(env_name, prefix): args = parser.parse_args() # Remove conda environments with a specific prefix - conda_source = "source " + os.path.join(args.conda_path, "etc/profile.d/conda.sh") + conda_source = ". " + os.path.join(args.conda_path, "etc/profile.d/conda.sh") check_env = conda_source + " && " + "conda env list" try: conda_envs = subprocess.run(check_env.split(" "), check=True, capture_output=True) diff --git a/harness/context_manager.py b/harness/context_manager.py index 923c98e9..b42847ee 100644 --- a/harness/context_manager.py +++ b/harness/context_manager.py @@ -281,7 +281,7 @@ def __enter__(self): # Install dependencies path_to_reqs = get_requirements(setup_ref_instance, self.testbed) - cmd = f"source {path_activate} {env_name} && echo 'activate successful' && pip install -r {path_to_reqs}" + cmd = f". {path_activate} {env_name} && echo 'activate successful' && pip install -r {path_to_reqs}" logger_testbed.info( f"[Testbed] Installing dependencies for {env_name}; Command: {cmd}" ) @@ -326,7 +326,7 @@ def __enter__(self): # Install additional packages if specified if "pip_packages" in install: - cmd = f"source {path_activate} {env_name} && pip install {install['pip_packages']}" + cmd = f". {path_activate} {env_name} && pip install {install['pip_packages']}" logger_testbed.info( f"[Testbed] Installing pip packages for {env_name}; Command: {cmd}" ) @@ -436,10 +436,7 @@ def __init__( ) self.log_file = os.path.join(log_dir, log_file_name) - self.cmd_activate = ( - f"source {os.path.join(self.conda_path, 'bin', 'activate')} " - + f"{self.venv} && echo 'activate successful'" - ) + self.cmd_activate = f". {os.path.join(self.conda_path, 'bin', 'activate')} {self.venv} && echo 'activate successful'" self.timeout = timeout shellenv = os.environ.copy() diff --git a/versioning/get_versions.py b/versioning/get_versions.py index f10d6d78..24ba2906 100644 --- a/versioning/get_versions.py +++ b/versioning/get_versions.py @@ -140,8 +140,8 @@ def get_versions_from_build(data: Dict): data["save_path"], ) # Activate conda environment and set installation command - cmd_activate = f"source {os.path.join(path_conda, 'bin/activate')}" - cmd_source = f"source {os.path.join(path_conda, 'etc/profile.d/conda.sh')}" + cmd_activate = f". {os.path.join(path_conda, 'bin/activate')}" + cmd_source = f". {os.path.join(path_conda, 'etc/profile.d/conda.sh')}" cmd_install = INSTALL_CMD[data_tasks[0]["repo"]] # Change directory to repo testbed From 39d43ed273dad3d68f27edfa2afdb24eced95c8c Mon Sep 17 00:00:00 2001 From: Xingyao Wang Date: Wed, 13 Mar 2024 22:39:12 -0500 Subject: [PATCH 2/3] add docker --- Dockerfile | 35 +++++++++++++++++++++++++++++++ harness/run_docker_interactive.sh | 13 ++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 Dockerfile create mode 100755 harness/run_docker_interactive.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2378da1f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM ubuntu:20.04 + +# https://github.com/princeton-nlp/SWE-bench/issues/15#issuecomment-1815392192 +RUN apt-get update && \ + apt-get install -y bash gcc git jq wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN git config --global user.email "swebench@pnlp.org" +RUN git config --global user.name "swebench" + +RUN apt update && apt install -y build-essential + +# Create new user +RUN useradd -ms /bin/bash swe-bench +USER swe-bench +WORKDIR /home/swe-bench + +# Setup Conda +ENV PATH="/home/swe-bench/miniconda3/bin:${PATH}" +ARG PATH="/home/swe-bench/miniconda3/bin:${PATH}" +RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh \ + && mkdir ~/.conda \ + && bash miniconda.sh -b \ + && rm -f miniconda.sh +RUN conda --version + +# Setup SWE-Bench Env +COPY environment.yml . +RUN conda env create -f environment.yml + +# Some missing packages +RUN pip install datasets python-dotenv gitpython + +CMD ["/bin/bash"] diff --git a/harness/run_docker_interactive.sh b/harness/run_docker_interactive.sh new file mode 100755 index 00000000..64020c84 --- /dev/null +++ b/harness/run_docker_interactive.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +DOCKER_IMAGE=xingyaoww/swe-bench +WORK_DIR=`pwd` + +docker run \ + -it \ + --rm \ + --user $(id -u):$(id -g) \ + --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 \ + -v $WORK_DIR:/swe-bench \ + $DOCKER_IMAGE \ + bash -c "cd /swe-bench && bash" From 8653c7a8adce1ad2409f5468b3af776d1443ab58 Mon Sep 17 00:00:00 2001 From: Xingyao Wang Date: Wed, 13 Mar 2024 22:39:58 -0500 Subject: [PATCH 3/3] fix matplotlib --- harness/constants.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/harness/constants.py b/harness/constants.py index 874dec86..8635b05b 100644 --- a/harness/constants.py +++ b/harness/constants.py @@ -165,6 +165,7 @@ "python": "3.9", "packages": "environment.yml", "install": "python -m pip install -e .", + "pip_packages": "pytest", } for k in ["3.5", "3.6", "3.7"] } @@ -174,6 +175,7 @@ "python": "3.8", "packages": "requirements.txt", "install": "python -m pip install -e .", + "pip_packages": "pytest" } for k in ["3.1", "3.2", "3.3", "3.4"] } @@ -184,6 +186,7 @@ "python": "3.7", "packages": "requirements.txt", "install": "python -m pip install -e .", + "pip_packages": "pytest", } for k in ["3.0"] } @@ -193,6 +196,7 @@ k: { "python": "3.5", "install": "python setup.py build; python setup.py install", + "pip_packages": "pytest", } for k in ["2.0", "2.1", "2.2", "1.0", "1.1", "1.2", "1.3", "1.4", "1.5"] } @@ -472,4 +476,4 @@ # Constants - Miscellaneous NON_TEST_EXTS = [".json", ".png", "csv", ".txt", ".md", ".jpg", ".jpeg", ".pkl", ".yml", ".yaml", ".toml"] -SWE_BENCH_URL_RAW = "https://raw.githubusercontent.com/" \ No newline at end of file +SWE_BENCH_URL_RAW = "https://raw.githubusercontent.com/"