Merge pull request #1 from xingyaoww/main

Merge some SWE-Bench fixes

Dockerfile

@@ -0,0 +1,35 @@
+FROM ubuntu:20.04
+
+# https://github.com/princeton-nlp/SWE-bench/issues/15#issuecomment-1815392192
+RUN apt-get update && \
+    apt-get install -y bash gcc git jq wget && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN git config --global user.email "[email protected]"
+RUN git config --global user.name "swebench"
+
+RUN apt update && apt install -y build-essential
+
+# Create new user
+RUN useradd -ms /bin/bash swe-bench
+USER swe-bench
+WORKDIR /home/swe-bench
+
+# Setup Conda
+ENV PATH="/home/swe-bench/miniconda3/bin:${PATH}"
+ARG PATH="/home/swe-bench/miniconda3/bin:${PATH}"
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh \
+    && mkdir ~/.conda \
+    && bash miniconda.sh -b \
+    && rm -f miniconda.sh
+RUN conda --version
+
+# Setup SWE-Bench Env
+COPY environment.yml .
+RUN conda env create -f environment.yml
+
+# Some missing packages
+RUN pip install datasets python-dotenv gitpython
+
+CMD ["/bin/bash"]

collect/cleanup/remove_envs.py

@@ -71,7 +71,7 @@ def remove_environment(env_name, prefix):
     args = parser.parse_args()
 
     # Remove conda environments with a specific prefix
-    conda_source = "source " + os.path.join(args.conda_path, "etc/profile.d/conda.sh")
+    conda_source = ". " + os.path.join(args.conda_path, "etc/profile.d/conda.sh")
     check_env = conda_source + " && " + "conda env list"
     try:
         conda_envs = subprocess.run(check_env.split(" "), check=True, capture_output=True)

harness/constants.py

@@ -165,6 +165,7 @@
         "python": "3.9",
         "packages": "environment.yml",
         "install": "python -m pip install -e .",
+        "pip_packages": "pytest",
     }
     for k in ["3.5", "3.6", "3.7"]
 }
@@ -174,6 +175,7 @@
             "python": "3.8",
             "packages": "requirements.txt",
             "install": "python -m pip install -e .",
+            "pip_packages": "pytest"
         }
         for k in ["3.1", "3.2", "3.3", "3.4"]
     }
@@ -184,6 +186,7 @@
             "python": "3.7",
             "packages": "requirements.txt",
             "install": "python -m pip install -e .",
+            "pip_packages": "pytest",
         }
         for k in ["3.0"]
     }
@@ -193,6 +196,7 @@
         k: {
             "python": "3.5",
             "install": "python setup.py build; python setup.py install",
+            "pip_packages": "pytest",
         }
         for k in ["2.0", "2.1", "2.2", "1.0", "1.1", "1.2", "1.3", "1.4", "1.5"]
     }
@@ -472,4 +476,4 @@
 
 # Constants - Miscellaneous
 NON_TEST_EXTS = [".json", ".png", "csv", ".txt", ".md", ".jpg", ".jpeg", ".pkl", ".yml", ".yaml", ".toml"]
-SWE_BENCH_URL_RAW = "https://raw.githubusercontent.com/"
+SWE_BENCH_URL_RAW = "https://raw.githubusercontent.com/"

harness/context_manager.py

@@ -284,7 +284,7 @@ def __enter__(self):
 
                     # Install dependencies
                     path_to_reqs = get_requirements(setup_ref_instance, self.testbed)
-                    cmd = f"source {path_activate} {env_name} && echo 'activate successful' && pip install -r {path_to_reqs}"
+                    cmd = f". {path_activate} {env_name} && echo 'activate successful' && pip install -r {path_to_reqs}"
                     logger_testbed.info(
                         f"[Testbed] Installing dependencies for {env_name}; Command: {cmd}"
                     )
@@ -329,7 +329,7 @@ def __enter__(self):
 
                 # Install additional packages if specified
                 if "pip_packages" in install:
-                    cmd = f"source {path_activate} {env_name} && pip install {install['pip_packages']}"
+                    cmd = f". {path_activate} {env_name} && pip install {install['pip_packages']}"
                     logger_testbed.info(
                         f"[Testbed] Installing pip packages for {env_name}; Command: {cmd}"
                     )
@@ -439,10 +439,7 @@ def __init__(
             )
         self.log_file = os.path.join(log_dir, log_file_name)
 
-        self.cmd_activate = (
-            f"source {os.path.join(self.conda_path, 'bin', 'activate')} "
-            + f"{self.venv} && echo 'activate successful'"
-        )
+        self.cmd_activate = f". {os.path.join(self.conda_path, 'bin', 'activate')} {self.venv} && echo 'activate successful'"
         self.timeout = timeout
 
         shellenv = os.environ.copy()

harness/run_docker_interactive.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+DOCKER_IMAGE=xingyaoww/swe-bench
+WORK_DIR=`pwd`
+
+docker run \
+    -it \
+    --rm \
+    --user $(id -u):$(id -g) \
+    --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 \
+    -v $WORK_DIR:/swe-bench \
+    $DOCKER_IMAGE \
+    bash -c "cd /swe-bench && bash"

versioning/get_versions.py

@@ -139,8 +139,8 @@ def get_versions_from_build(data: dict):
         data["save_path"],
     )
     # Activate conda environment and set installation command
-    cmd_activate = f"source {os.path.join(path_conda, 'bin/activate')}"
-    cmd_source = f"source {os.path.join(path_conda, 'etc/profile.d/conda.sh')}"
+    cmd_activate = f". {os.path.join(path_conda, 'bin/activate')}"
+    cmd_source = f". {os.path.join(path_conda, 'etc/profile.d/conda.sh')}"
     cmd_install = INSTALL_CMD[data_tasks[0]["repo"]]
 
     # Change directory to repo testbed