[ClusterFuzzLite] Support GCB and gsutil/gcs as filestore. (google#6629)

* add gsutil filestore

* lint

* Fix

* Add build image script

* get gcb fuzzing working

* fmt and fix config_utils_test

* Check that crashes are uploaded

* Add no_filestore

* fix test

* fix tests

* fix

* Print crash URL

* Fix

* fix

* fmt

* lnt

* fix

* fmt

.dockerignore

@@ -1,4 +1,5 @@
 .git
+.venv
 infra/cifuzz/test_data/*
 docs/*
 
@@ -8,4 +9,6 @@ docs/*
 build
 *~
 .DS_Store
-*.swp
+*.swp
+.pytest_cache
+*__pycache__/*

infra/cifuzz/build-images.sh

@@ -0,0 +1,28 @@
+#! /bin/bash -eux
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script for building the docker images for cifuzz.
+
+CIFUZZ_DIR=$(dirname "$0")
+CIFUZZ_DIR=$(realpath $CIFUZZ_DIR)
+INFRA_DIR=$(realpath $CIFUZZ_DIR/..)
+OSS_FUZZ_ROOT=$(realpath $INFRA_DIR/..)
+
+# Build cifuzz-base.
+docker build --tag gcr.io/oss-fuzz-base/cifuzz-base --file $CIFUZZ_DIR/cifuzz-base/Dockerfile $OSS_FUZZ_ROOT
+
+# Build run-fuzzers and build-fuzzers images.
+docker build --tag gcr.io/oss-fuzz-base/cifuzz-build-fuzzers:v1 --file $INFRA_DIR/build_fuzzers.Dockerfile $INFRA_DIR
+docker build --tag gcr.io/oss-fuzz-base/cifuzz-run-fuzzers:v1 --file $INFRA_DIR/run_fuzzers.Dockerfile $INFRA_DIR

infra/cifuzz/build_fuzzers_test.py

@@ -201,6 +201,7 @@ def test_external_generic_project(self):
         project_repo_name=project_repo_name,
         workspace=self.workspace,
         git_url=git_url,
+        filestore='no_filestore',
         commit_sha='HEAD',
         project_src_path=project_src_path,
         base_commit='HEAD^1')

infra/cifuzz/cifuzz-base/Dockerfile

@@ -20,8 +20,17 @@ RUN apt-get update && \
     apt-get install -y systemd && \
     apt-get install -y --no-install-recommends nodejs npm && \
     wget https://download.docker.com/linux/ubuntu/dists/focal/pool/stable/amd64/docker-ce-cli_20.10.8~3-0~ubuntu-focal_amd64.deb -O /tmp/docker-ce.deb && \
-    dpkg -i /tmp/docker-ce.deb && rm /tmp/docker-ce.deb
+    dpkg -i /tmp/docker-ce.deb && \
+    rm /tmp/docker-ce.deb && \
+    mkdir -p /opt/gcloud && \
+    wget -qO- https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz | tar zxv -C /opt/gcloud && \
+    /opt/gcloud/google-cloud-sdk/install.sh --usage-reporting=false --bash-completion=false --disable-installation-options && \
+    apt-get -y install gcc python3-dev && \
+    pip3 install -U crcmod && \
+    apt-get autoremove -y gcc python3-dev
 
+
+ENV PATH=/opt/gcloud/google-cloud-sdk/bin/:$PATH
 ENV OSS_FUZZ_ROOT=/opt/oss-fuzz
 ADD . ${OSS_FUZZ_ROOT}
 RUN python3 -m pip install -r ${OSS_FUZZ_ROOT}/infra/cifuzz/requirements.txt

infra/cifuzz/cifuzz_end_to_end_test.py

@@ -39,6 +39,8 @@ def test_simple(self):
     """Simple end-to-end test using run_cifuzz.main()."""
     os.environ['REPOSITORY'] = 'external-project'
     os.environ['PROJECT_SRC_PATH'] = EXTERNAL_PROJECT_PATH
+    os.environ['FILESTORE'] = 'no_filestore'
+    os.environ['NO_CLUSTERFUZZ_DEPLOYMENT'] = 'True'
 
     with test_helpers.docker_temp_dir() as temp_dir:
       os.environ['WORKSPACE'] = temp_dir

infra/cifuzz/clusterfuzz_deployment.py

@@ -100,7 +100,7 @@ def download_latest_build(self):
       # called if multiple bugs are found.
       return self.workspace.clusterfuzz_build
 
-    repo_dir = self.ci_system.repo_dir()
+    repo_dir = self.ci_system.repo_dir
     if not repo_dir:
       raise RuntimeError('Repo checkout does not exist.')
 
@@ -355,20 +355,19 @@ def get_coverage(self, repo_path):
 
 
 _PLATFORM_CLUSTERFUZZ_DEPLOYMENT_MAPPING = {
-    config_utils.BaseConfig.Platform.INTERNAL_GENERIC_CI:
-        OSSFuzz,
-    config_utils.BaseConfig.Platform.INTERNAL_GITHUB:
-        OSSFuzz,
-    config_utils.BaseConfig.Platform.EXTERNAL_GENERIC_CI:
-        NoClusterFuzzDeployment,
-    config_utils.BaseConfig.Platform.EXTERNAL_GITHUB:
-        ClusterFuzzLite,
+    config_utils.BaseConfig.Platform.INTERNAL_GENERIC_CI: OSSFuzz,
+    config_utils.BaseConfig.Platform.INTERNAL_GITHUB: OSSFuzz,
+    config_utils.BaseConfig.Platform.EXTERNAL_GENERIC_CI: ClusterFuzzLite,
+    config_utils.BaseConfig.Platform.EXTERNAL_GITHUB: ClusterFuzzLite,
 }
 
 
 def get_clusterfuzz_deployment(config, workspace):
   """Returns object reprsenting deployment of ClusterFuzz used by |config|."""
   deployment_cls = _PLATFORM_CLUSTERFUZZ_DEPLOYMENT_MAPPING[config.platform]
+  if config.no_clusterfuzz_deployment:
+    logging.info('Overriding ClusterFuzzDeployment. Using None.')
+    deployment_cls = NoClusterFuzzDeployment
   result = deployment_cls(config, workspace)
   logging.info('ClusterFuzzDeployment: %s.', result)
   return result

infra/cifuzz/clusterfuzz_deployment_test.py

@@ -132,6 +132,7 @@ def setUp(self):
     self.setUpPyfakefs()
     self.deployment = _create_deployment(run_fuzzers_mode='batch',
                                          oss_fuzz_project_name='',
+                                         cloud_bucket='gs://bucket',
                                          is_github=True)
     self.corpus_dir = os.path.join(self.deployment.workspace.corpora,
                                    EXAMPLE_FUZZER)
@@ -157,7 +158,7 @@ def test_download_corpus_fail(self, _):
               side_effect=[False, True])
   @mock.patch('repo_manager.RepoManager.get_commit_list',
               return_value=['commit1', 'commit2'])
-  @mock.patch('continuous_integration.BaseCi.repo_dir',
+  @mock.patch('continuous_integration.GithubCiMixin.repo_dir',
               return_value='/path/to/repo')
   def test_download_latest_build(self, mock_repo_dir, mock_get_commit_list,
                                  mock_download_build):
@@ -173,7 +174,7 @@ def test_download_latest_build(self, mock_repo_dir, mock_get_commit_list,
               side_effect=Exception)
   @mock.patch('repo_manager.RepoManager.get_commit_list',
               return_value=['commit1', 'commit2'])
-  @mock.patch('continuous_integration.BaseCi.repo_dir',
+  @mock.patch('continuous_integration.GithubCiMixin.repo_dir',
               return_value='/path/to/repo')
   def test_download_latest_build_fail(self, mock_repo_dir, mock_get_commit_list,
                                       _):
@@ -195,10 +196,13 @@ class NoClusterFuzzDeploymentTest(fake_filesystem_unittest.TestCase):
   def setUp(self):
     self.setUpPyfakefs()
     config = test_helpers.create_run_config(workspace=WORKSPACE,
-                                            is_github=False)
+                                            is_github=False,
+                                            filestore='no_filestore',
+                                            no_clusterfuzz_deployment=True)
     workspace = workspace_utils.Workspace(config)
     self.deployment = clusterfuzz_deployment.get_clusterfuzz_deployment(
         config, workspace)
+
     self.corpus_dir = os.path.join(workspace.corpora, EXAMPLE_FUZZER)
 
   @mock.patch('logging.info')
@@ -241,7 +245,7 @@ def setUp(self):
       (config_utils.BaseConfig.Platform.INTERNAL_GITHUB,
        clusterfuzz_deployment.OSSFuzz),
       (config_utils.BaseConfig.Platform.EXTERNAL_GENERIC_CI,
-       clusterfuzz_deployment.NoClusterFuzzDeployment),
+       clusterfuzz_deployment.ClusterFuzzLite),
       (config_utils.BaseConfig.Platform.EXTERNAL_GITHUB,
        clusterfuzz_deployment.ClusterFuzzLite),
   ])

infra/cifuzz/config_utils.py

@@ -236,7 +236,12 @@ def __init__(self):
     self.git_store_branch = os.environ.get('GIT_STORE_BRANCH')
     self.git_store_branch_coverage = os.environ.get('GIT_STORE_BRANCH_COVERAGE',
                                                     self.git_store_branch)
+    self.project_src_path = self._ci_env.project_src_path
     self.docker_in_docker = os.environ.get('DOCKER_IN_DOCKER')
+    self.filestore = os.environ.get('FILESTORE')
+    self.cloud_bucket = os.environ.get('CLOUD_BUCKET')
+    self.no_clusterfuzz_deployment = os.environ.get('NO_CLUSTERFUZZ_DEPLOYMENT',
+                                                    False)
 
     # TODO(metzman): Fix tests to create valid configurations and get rid of
     # CIFUZZ_TEST here and in presubmit.py.
@@ -261,6 +266,10 @@ def validate(self):
                     constants.LANGUAGES)
       return False
 
+    if not self.project_repo_name:
+      logging.error('Must set REPOSITORY.')
+      return False
+
     return True
 
   @property
@@ -361,7 +370,6 @@ def __init__(self):
     self._get_config_from_event_path(event)
 
     self.base_ref = os.getenv('GITHUB_BASE_REF')
-    self.project_src_path = self._ci_env.project_src_path
 
     self.allowed_broken_targets_percentage = os.getenv(
         'ALLOWED_BROKEN_TARGETS_PERCENTAGE')

infra/cifuzz/config_utils_test.py

@@ -91,6 +91,7 @@ def test_validate(self):
     """Tests that validate returns True if config is valid."""
     os.environ['OSS_FUZZ_PROJECT_NAME'] = 'example'
     os.environ['WORKSPACE'] = '/workspace'
+    os.environ['REPOSITORY'] = 'repo'
     config = self._create_config()
     self.assertTrue(config.validate())
 

infra/cifuzz/continuous_integration.py

@@ -53,23 +53,13 @@ class BaseCi:
   def __init__(self, config):
     self.config = config
     self.workspace = workspace_utils.Workspace(config)
+    self._repo_dir = None
 
+  @property
   def repo_dir(self):
     """Returns the source repo path, if it has been checked out. None is
     returned otherwise."""
-    if not os.path.exists(self.workspace.repo_storage):
-      return None
-
-    # Note: this assumes there is only one repo checked out here.
-    listing = os.listdir(self.workspace.repo_storage)
-    if len(listing) != 1:
-      raise RuntimeError('Invalid repo storage.')
-
-    repo_path = os.path.join(self.workspace.repo_storage, listing[0])
-    if not os.path.isdir(repo_path):
-      raise RuntimeError('Repo is not a directory.')
-
-    return repo_path
+    raise NotImplementedError('Child class must implement method.')
 
   def prepare_for_fuzzer_build(self):
     """Builds the fuzzer builder image and gets the source code we need to
@@ -152,6 +142,31 @@ def checkout_specified_commit(repo_manager_obj, pr_ref, commit_sha):
 class GithubCiMixin:
   """Mixin for Github based CI systems."""
 
+  def __init__(self, config):
+    super().__init__(config)
+    # Unlike in other classes, here _repo_dir is the parent directory of the
+    # repo, not its actual directory.
+    self._repo_dir = self.workspace.repo_storage
+
+  @property
+  def repo_dir(self):
+    """Returns the source repo path, if it has been checked out. None is
+    returned otherwise."""
+    if not os.path.exists(self._repo_dir):
+      logging.warning('Repo dir: %s does not exist.', self._repo_dir)
+      return None
+
+    # Note: this assumes there is only one repo checked out here.
+    listing = os.listdir(self._repo_dir)
+    if len(listing) != 1:
+      raise RuntimeError('Invalid repo directory.')
+
+    repo_path = os.path.join(self._repo_dir, listing[0])
+    if not os.path.isdir(repo_path):
+      raise RuntimeError('Repo is not a directory.')
+
+    return repo_path
+
   def get_diff_base(self):
     """Returns the base to diff against with git to get the change under
     test."""
@@ -217,6 +232,16 @@ class InternalGeneric(BaseCi):
   """Class representing CI for an OSS-Fuzz project on a CI other than Github
   actions."""
 
+  def __init__(self, config):
+    super().__init__(config)
+    self._repo_dir = config.project_src_path
+
+  @property
+  def repo_dir(self):
+    """Returns the source repo path, if it has been checked out. None is
+    returned otherwise."""
+    return self._repo_dir
+
   def prepare_for_fuzzer_build(self):
     """Builds the project builder image for an OSS-Fuzz project outside of
     GitHub actions. Returns the repo_manager. Does not checkout source code
@@ -263,6 +288,16 @@ def build_external_project_docker_image(project_src, build_integration_path):
 class ExternalGeneric(BaseCi):
   """CI implementation for generic CI for external (non-OSS-Fuzz) projects."""
 
+  def __init__(self, config):
+    super().__init__(config)
+    self._repo_dir = config.project_src_path
+
+  @property
+  def repo_dir(self):
+    """Returns the source repo path, if it has been checked out. None is
+    returned otherwise."""
+    return self._repo_dir
+
   def get_diff_base(self):
     return 'origin...'
 

infra/cifuzz/filestore/__init__.py

@@ -49,6 +49,6 @@ def download_build(self, name, dst_directory):
     """Downloads the build with |name| to |dst_directory|."""
     raise NotImplementedError('Child class must implement method.')
 
-  def download_coverage(self, dst_directory):
+  def download_coverage(self, name, dst_directory):
     """Downloads the latest project coverage report."""
     raise NotImplementedError('Child class must implement method.')

infra/cifuzz/filestore/github_actions/__init__.py

@@ -21,8 +21,9 @@
 
 # pylint: disable=wrong-import-position,import-error
 sys.path.append(
-    os.path.join(os.path.pardir, os.path.pardir, os.path.pardir,
-                 os.path.dirname(os.path.abspath(__file__))))
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir,
+                     os.path.pardir)))
 
 import utils
 import http_utils

infra/cifuzz/filestore/github_actions/github_actions_test.py

@@ -24,8 +24,7 @@
 
 # pylint: disable=wrong-import-position
 INFRA_DIR = os.path.dirname(
-    os.path.dirname(os.path.dirname(os.path.dirname(
-        os.path.abspath(__file__)))))
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 sys.path.append(INFRA_DIR)
 
 from filestore import github_actions

infra/cifuzz/filestore/github_actions/github_api_test.py

@@ -12,8 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Tests for github_api."""
+import os
+import sys
 import unittest
 
+# pylint: disable=wrong-import-position,import-error
+sys.path.append(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir,
+                     os.path.pardir)))
+
 from filestore.github_actions import github_api
 import test_helpers