[nss-coverage] Pull corpora directly from bucket MozillaSecurity#557

services/nss-coverage/Dockerfile

@@ -16,7 +16,6 @@ COPY recipes/linux/ /src/recipes/
 COPY services/fuzzing-decision /src/fuzzing-tc
 COPY services/nss-coverage/setup.sh /src/recipes/setup-nss-coverage.sh
 COPY \
-    services/nss-coverage/libfuzzer-options.py \
     services/nss-coverage/merge-coverage.py \
     services/nss-coverage/nspr_map.json \
     services/nss-coverage/launch-worker.sh \

services/nss-coverage/launch-worker.sh

@@ -62,40 +62,12 @@ if [[ ! -d cryptofuzz ]]; then
   git-clone https://github.com/MozillaSecurity/cryptofuzz.git
 fi
 
-COVRUNTIME=${COVRUNTIME-3600}
-
-function clone-corpus {
-  local name=$1
-  local url=$2
-  shift 2
-
-  mkdir -p corpus
-  pushd corpus
-  if [[ ! -d "$name" ]]; then
-    mkdir "$name"
-    pushd "$name"
-
-    # There may be no OSS-Fuzz corpus yet for new fuzz targets
-    code=$(retry-curl --no-fail -w "%{http_code}" -O "$url")
-    if [[ $code -eq 200 ]]; then
-      unzip public.zip
-    else
-      echo "WARNING - cloning corpus for $name failed with code: $code" >&2
-    fi
-    rm public.zip
-
-    popd
-  fi
-  popd
-}
+# Setup gcloud
+mkdir -p ~/.config/gcloud
+get-tc-secret ossfuzz-gutils ~/.config/gcloud/application_default_credentials.json raw
+echo -e "[Credentials]\ngs_service_key_file = /home/worker/.config/gcloud/application_default_credentials.json" > .boto
 
-function clone-nssfuzz-corpus {
-  local name="$1"
-  shift 1
-
-  clone-corpus "$name" \
-               "https://storage.googleapis.com/nss-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/nss_$name/public.zip"
-}
+COVRUNTIME=${COVRUNTIME-3600}
 
 function run-target {
   local target="$1"
@@ -137,66 +109,48 @@ function run-target {
   fi
 }
 
-function run-nssfuzz-target {
-  local target="$1"
-  local name="$2"
-  shift 2
-
-  readarray -t options < <(python libfuzzer-options.py nss/fuzz/options/"$name".options)
-  run-target "dist/Debug/bin/nssfuzz-$target" "$name" "${options[@]}"
-}
+# Build nss w/o tls fuzzing mode
+update-status "building nss w/o tls fuzzing mode"
+pushd nss
+time ./build.sh -c -v --fuzz --disable-tests
+popd
 
-declare -A targets=()
-declare -A tls_targets=()
+for fuzzer in dist/Debug/bin/nssfuzz-*; do
+  file="$(basename "$fuzzer")"
+  name="${file#nssfuzz-}"
 
-for file in nss/fuzz/options/*; do
-  name="$(basename "$file" .options)"
-  if [[ "$name" =~ -no_fuzzer_mode$ ]]; then
-    tls_targets["${name%-no_fuzzer_mode}"]=1
-    continue
+  if [[ -f "nss/fuzz/options/$name-no_fuzzer_mode.options" ]]; then
+    name="${name}-no_fuzzer_mode"
   fi
 
-  targets["$name"]=1
-done
+  update-status "cloning corpus for target $name"
+  mkdir -p "./corpus/$name"
+  gsutil -m cp -r "gs://nss-corpus.clusterfuzz-external.appspot.com/libFuzzer/nss_$name" "./corpus/$name"
 
-total_targets=$(("${#targets[@]}" + "${#tls_targets[@]}"))
-curr_target_n=1
+  update-status "running target $name"
+  readarray -t options < <(python "nss/fuzz/config/libfuzzer-options.py nss/fuzz/options/$name.options")
+  run-target "$fuzzer" "$name" "${options[@]}"
+done
 
 # Build nss with tls fuzzing mode
-update-status "building nss with tls fuzzing mode ($curr_target_n/$total_targets have run)"
+update-status "building nss with tls fuzzing mode"
 pushd nss
 time ./build.sh -c -v --fuzz=tls --disable-tests
 popd
 
-# For each nssfuzz target with tls fuzzing mode, clone corpus & run
-for target in "${!tls_targets[@]}"; do
-  update-status "cloning corpus for $target ($curr_target_n/$total_targets)"
-  clone-nssfuzz-corpus "$target"
-
-  update-status "running $target ($curr_target_n/$total_targets)"
-  run-nssfuzz-target "$target" "$target"
-  ((curr_target_n++))
-done
+for fuzzer in dist/Debug/bin/nssfuzz-*; do
+  file="$(basename "$fuzzer")"
+  name="${file#nssfuzz-}"
 
-# Build nss w/o tls fuzzing mode
-update-status "building nss w/o tls fuzzing mode"
-pushd nss
-time ./build.sh -c -v --fuzz --disable-tests
-popd
+  if [[ -f "nss/fuzz/options/$name-no_fuzzer_mode.options" ]]; then
+    update-status "cloning corpus for target $name"
+    mkdir -p "./corpus/$name"
+    gsutil -m cp -r "gs://nss-corpus.clusterfuzz-external.appspot.com/libFuzzer/nss_$name" "./corpus/$name"
 
-# For each nssfuzz target w/o tls fuzzing mode, clone corpus & run
-for target in "${!targets[@]}"; do
-  name="$target"
-  if [[ -n "${tls_targets[$target]:-}" ]]; then
-    name="$name-no_fuzzer_mode"
+    update-status "running target $name"
+    readarray -t options < <(python "nss/fuzz/config/libfuzzer-options.py nss/fuzz/options/$name.options")
+    run-target "$fuzzer" "$name" "${options[@]}"
   fi
-
-  update-status "cloning corpus for $name ($curr_target_n/$total_targets)"
-  clone-nssfuzz-corpus "$name"
-
-  update-status "running $name ($curr_target_n/$total_targets)"
-  run-nssfuzz-target "$target" "$name"
-  ((curr_target_n++))
 done
 
 # Generate cryptofuzz headers
@@ -224,8 +178,13 @@ popd
 
 # Clone cryptofuzz nss corpus
 update-status "cloning cryptofuzz nss corpus"
-clone-corpus "cryptofuzz" \
-             "https://storage.googleapis.com/cryptofuzz-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/cryptofuzz_cryptofuzz-nss/public.zip"
+mkdir -p ./corpus/cryptofuzz
+
+pushd ./corpus/cryptofuzz
+retry-curl -O "https://storage.googleapis.com/cryptofuzz-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/cryptofuzz_cryptofuzz-nss/public.zip"
+unzip public.zip
+rm -f public.zip
+popd
 
 # Run cryptofuzz
 update-status "running cryptofuzz"

services/nss-coverage/setup.sh

@@ -22,6 +22,7 @@ EDIT=1 SRCDIR=/src/fuzzing-tc ./fuzzing_tc.sh
 ./fuzzmanager.sh
 ./grcov.sh
 ./taskcluster.sh
+./gsutil.sh
 
 packages=(
   binutils