From 5b454314725d72a263cb413466e6966a7fb30763 Mon Sep 17 00:00:00 2001
From: moz-mdauer <mdauer@mozilla.com>
Date: Mon, 20 Jan 2025 13:33:02 +0100
Subject: [PATCH] [nss-coverage] Pull corpora directly from bucket #557

---
 services/nss-coverage/Dockerfile           |   1 -
 services/nss-coverage/launch-worker.sh     | 121 +++++++--------------
 services/nss-coverage/libfuzzer-options.py |  19 ----
 services/nss-coverage/setup.sh             |   1 +
 4 files changed, 39 insertions(+), 103 deletions(-)
 delete mode 100644 services/nss-coverage/libfuzzer-options.py

diff --git a/services/nss-coverage/Dockerfile b/services/nss-coverage/Dockerfile
index e0e1581a..6e59701a 100644
--- a/services/nss-coverage/Dockerfile
+++ b/services/nss-coverage/Dockerfile
@@ -16,7 +16,6 @@ COPY recipes/linux/ /src/recipes/
 COPY services/fuzzing-decision /src/fuzzing-tc
 COPY services/nss-coverage/setup.sh /src/recipes/setup-nss-coverage.sh
 COPY \
-    services/nss-coverage/libfuzzer-options.py \
     services/nss-coverage/merge-coverage.py \
     services/nss-coverage/nspr_map.json \
     services/nss-coverage/launch-worker.sh \
diff --git a/services/nss-coverage/launch-worker.sh b/services/nss-coverage/launch-worker.sh
index 28c8711d..a50adfbe 100755
--- a/services/nss-coverage/launch-worker.sh
+++ b/services/nss-coverage/launch-worker.sh
@@ -62,40 +62,12 @@ if [[ ! -d cryptofuzz ]]; then
   git-clone https://github.com/MozillaSecurity/cryptofuzz.git
 fi
 
-COVRUNTIME=${COVRUNTIME-3600}
-
-function clone-corpus {
-  local name=$1
-  local url=$2
-  shift 2
+# Setup gcloud
+mkdir -p ~/.config/gcloud
+get-tc-secret ossfuzz-gutils ~/.config/gcloud/application_default_credentials.json raw
+echo -e "[Credentials]\ngs_service_key_file = /home/worker/.config/gcloud/application_default_credentials.json" > .boto
 
-  mkdir -p corpus
-  pushd corpus
-  if [[ ! -d "$name" ]]; then
-    mkdir "$name"
-    pushd "$name"
-
-    # There may be no OSS-Fuzz corpus yet for new fuzz targets
-    code=$(retry-curl --no-fail -w "%{http_code}" -O "$url")
-    if [[ $code -eq 200 ]]; then
-      unzip public.zip
-    else
-      echo "WARNING - cloning corpus for $name failed with code: $code" >&2
-    fi
-    rm public.zip
-
-    popd
-  fi
-  popd
-}
-
-function clone-nssfuzz-corpus {
-  local name="$1"
-  shift 1
-
-  clone-corpus "$name" \
-               "https://storage.googleapis.com/nss-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/nss_$name/public.zip"
-}
+COVRUNTIME=${COVRUNTIME-3600}
 
 function run-target {
   local target="$1"
@@ -137,66 +109,44 @@ function run-target {
   fi
 }
 
-function run-nssfuzz-target {
-  local target="$1"
-  local name="$2"
-  shift 2
-
-  readarray -t options < <(python libfuzzer-options.py nss/fuzz/options/"$name".options)
-  run-target "dist/Debug/bin/nssfuzz-$target" "$name" "${options[@]}"
-}
-
-declare -A targets=()
-declare -A tls_targets=()
+# Build nss w/o tls fuzzing mode
+update-status "building nss w/o tls fuzzing mode"
+pushd nss
+time ./build.sh -c -v --fuzz --disable-tests
+popd
 
-for file in nss/fuzz/options/*; do
-  name="$(basename "$file" .options)"
-  if [[ "$name" =~ -no_fuzzer_mode$ ]]; then
-    tls_targets["${name%-no_fuzzer_mode}"]=1
-    continue
+for fuzzer in dist/Debug/bin/nssfuzz-*; do
+  name="$(basename "$fuzzer")"
+  if [[ -f "nss/fuzz/options/$name-no_fuzzer_mode.options" ]]; then
+    name="${name}-no_fuzzer_mode"
   fi
 
-  targets["$name"]=1
-done
+  update-status "cloning corpus for target $name"
+  mkdir -p "./corpus/$name"
+  gsutil -m cp -r "gs://nss-corpus.clusterfuzz-external.appspot.com/libFuzzer/nss_$name" "./corpus/$name"
 
-total_targets=$(("${#targets[@]}" + "${#tls_targets[@]}"))
-curr_target_n=1
+  update-status "running target $name"
+  readarray -t options < <(python "nss/fuzz/config/libfuzzer-options.py nss/fuzz/options/$name.options")
+  run-target "$fuzzer" "$name" "${options[@]}"
+done
 
 # Build nss with tls fuzzing mode
-update-status "building nss with tls fuzzing mode ($curr_target_n/$total_targets have run)"
+update-status "building nss w/o tls fuzzing mode"
 pushd nss
 time ./build.sh -c -v --fuzz=tls --disable-tests
 popd
 
-# For each nssfuzz target with tls fuzzing mode, clone corpus & run
-for target in "${!tls_targets[@]}"; do
-  update-status "cloning corpus for $target ($curr_target_n/$total_targets)"
-  clone-nssfuzz-corpus "$target"
+for fuzzer in dist/Debug/bin/nssfuzz-*; do
+  name="$(basename "$fuzzer")"
+  if [[ -f "nss/fuzz/options/$name-no_fuzzer_mode.options" ]]; then
+    update-status "cloning corpus for target $name"
+    mkdir -p "./corpus/$name"
+    gsutil -m cp -r "gs://nss-corpus.clusterfuzz-external.appspot.com/libFuzzer/nss_$name" "./corpus/$name"
 
-  update-status "running $target ($curr_target_n/$total_targets)"
-  run-nssfuzz-target "$target" "$target"
-  ((curr_target_n++))
-done
-
-# Build nss w/o tls fuzzing mode
-update-status "building nss w/o tls fuzzing mode"
-pushd nss
-time ./build.sh -c -v --fuzz --disable-tests
-popd
-
-# For each nssfuzz target w/o tls fuzzing mode, clone corpus & run
-for target in "${!targets[@]}"; do
-  name="$target"
-  if [[ -n "${tls_targets[$target]:-}" ]]; then
-    name="$name-no_fuzzer_mode"
+    update-status "running target $name"
+    readarray -t options < <(python "nss/fuzz/config/libfuzzer-options.py nss/fuzz/options/$name.options")
+    run-target "$fuzzer" "$name" "${options[@]}"
   fi
-
-  update-status "cloning corpus for $name ($curr_target_n/$total_targets)"
-  clone-nssfuzz-corpus "$name"
-
-  update-status "running $name ($curr_target_n/$total_targets)"
-  run-nssfuzz-target "$target" "$name"
-  ((curr_target_n++))
 done
 
 # Generate cryptofuzz headers
@@ -224,8 +174,13 @@ popd
 
 # Clone cryptofuzz nss corpus
 update-status "cloning cryptofuzz nss corpus"
-clone-corpus "cryptofuzz" \
-             "https://storage.googleapis.com/cryptofuzz-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/cryptofuzz_cryptofuzz-nss/public.zip"
+mkdir -p ./corpus/cryptofuzz
+
+pushd ./corpus/cryptofuzz
+retry-curl -O "https://storage.googleapis.com/cryptofuzz-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/cryptofuzz_cryptofuzz-nss/public.zip"
+unzip public.zip
+rm -f public.zip
+popd
 
 # Run cryptofuzz
 update-status "running cryptofuzz"
diff --git a/services/nss-coverage/libfuzzer-options.py b/services/nss-coverage/libfuzzer-options.py
deleted file mode 100644
index 712acc91..00000000
--- a/services/nss-coverage/libfuzzer-options.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at https://mozilla.org/MPL/2.0/.
-
-import sys
-import toml
-
-
-def main():
-    with open(sys.argv[1], "r") as f:
-        data = toml.load(f)
-
-    for key, value in data["libfuzzer"].items():
-        print(f"-{key}={value}")
-
-
-if __name__ == "__main__":
-    main()
diff --git a/services/nss-coverage/setup.sh b/services/nss-coverage/setup.sh
index 33fa9e91..e9a6fd4e 100755
--- a/services/nss-coverage/setup.sh
+++ b/services/nss-coverage/setup.sh
@@ -22,6 +22,7 @@ EDIT=1 SRCDIR=/src/fuzzing-tc ./fuzzing_tc.sh
 ./fuzzmanager.sh
 ./grcov.sh
 ./taskcluster.sh
+./gsutil.sh
 
 packages=(
   binutils