Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[nss-coverage] Pull corpora directly from bucket #557 #593

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion services/nss-coverage/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ COPY recipes/linux/ /src/recipes/
COPY services/fuzzing-decision /src/fuzzing-tc
COPY services/nss-coverage/setup.sh /src/recipes/setup-nss-coverage.sh
COPY \
services/nss-coverage/libfuzzer-options.py \
services/nss-coverage/merge-coverage.py \
services/nss-coverage/nspr_map.json \
services/nss-coverage/launch-worker.sh \
Expand Down
121 changes: 40 additions & 81 deletions services/nss-coverage/launch-worker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -62,40 +62,12 @@ if [[ ! -d cryptofuzz ]]; then
git-clone https://github.com/MozillaSecurity/cryptofuzz.git
fi

COVRUNTIME=${COVRUNTIME-3600}

function clone-corpus {
local name=$1
local url=$2
shift 2

mkdir -p corpus
pushd corpus
if [[ ! -d "$name" ]]; then
mkdir "$name"
pushd "$name"

# There may be no OSS-Fuzz corpus yet for new fuzz targets
code=$(retry-curl --no-fail -w "%{http_code}" -O "$url")
if [[ $code -eq 200 ]]; then
unzip public.zip
else
echo "WARNING - cloning corpus for $name failed with code: $code" >&2
fi
rm public.zip

popd
fi
popd
}
# Setup gcloud
mkdir -p ~/.config/gcloud
get-tc-secret ossfuzz-gutils ~/.config/gcloud/application_default_credentials.json raw
echo -e "[Credentials]\ngs_service_key_file = /home/worker/.config/gcloud/application_default_credentials.json" > .boto

function clone-nssfuzz-corpus {
local name="$1"
shift 1

clone-corpus "$name" \
"https://storage.googleapis.com/nss-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/nss_$name/public.zip"
}
COVRUNTIME=${COVRUNTIME-3600}

function run-target {
local target="$1"
Expand Down Expand Up @@ -137,66 +109,48 @@ function run-target {
fi
}

function run-nssfuzz-target {
local target="$1"
local name="$2"
shift 2

readarray -t options < <(python libfuzzer-options.py nss/fuzz/options/"$name".options)
run-target "dist/Debug/bin/nssfuzz-$target" "$name" "${options[@]}"
}
# Build nss w/o tls fuzzing mode
update-status "building nss w/o tls fuzzing mode"
pushd nss
time ./build.sh -c -v --fuzz --disable-tests
popd

declare -A targets=()
declare -A tls_targets=()
for fuzzer in dist/Debug/bin/nssfuzz-*; do
file="$(basename "$fuzzer")"
name="${file#nssfuzz-}"

for file in nss/fuzz/options/*; do
name="$(basename "$file" .options)"
if [[ "$name" =~ -no_fuzzer_mode$ ]]; then
tls_targets["${name%-no_fuzzer_mode}"]=1
continue
if [[ -f "nss/fuzz/options/$name-no_fuzzer_mode.options" ]]; then
name="${name}-no_fuzzer_mode"
fi

targets["$name"]=1
done
update-status "cloning corpus for target $name"
mkdir -p "./corpus/$name"
gsutil -m cp -r "gs://nss-corpus.clusterfuzz-external.appspot.com/libFuzzer/nss_$name" "./corpus/$name"

total_targets=$(("${#targets[@]}" + "${#tls_targets[@]}"))
curr_target_n=1
update-status "running target $name"
readarray -t options < <(python "nss/fuzz/config/libfuzzer-options.py nss/fuzz/options/$name.options")
run-target "$fuzzer" "$name" "${options[@]}"
done

# Build nss with tls fuzzing mode
update-status "building nss with tls fuzzing mode ($curr_target_n/$total_targets have run)"
update-status "building nss with tls fuzzing mode"
pushd nss
time ./build.sh -c -v --fuzz=tls --disable-tests
popd

# For each nssfuzz target with tls fuzzing mode, clone corpus & run
for target in "${!tls_targets[@]}"; do
update-status "cloning corpus for $target ($curr_target_n/$total_targets)"
clone-nssfuzz-corpus "$target"

update-status "running $target ($curr_target_n/$total_targets)"
run-nssfuzz-target "$target" "$target"
((curr_target_n++))
done
for fuzzer in dist/Debug/bin/nssfuzz-*; do
file="$(basename "$fuzzer")"
name="${file#nssfuzz-}"

# Build nss w/o tls fuzzing mode
update-status "building nss w/o tls fuzzing mode"
pushd nss
time ./build.sh -c -v --fuzz --disable-tests
popd
if [[ -f "nss/fuzz/options/$name-no_fuzzer_mode.options" ]]; then
update-status "cloning corpus for target $name"
mkdir -p "./corpus/$name"
gsutil -m cp -r "gs://nss-corpus.clusterfuzz-external.appspot.com/libFuzzer/nss_$name" "./corpus/$name"

# For each nssfuzz target w/o tls fuzzing mode, clone corpus & run
for target in "${!targets[@]}"; do
name="$target"
if [[ -n "${tls_targets[$target]:-}" ]]; then
name="$name-no_fuzzer_mode"
update-status "running target $name"
readarray -t options < <(python "nss/fuzz/config/libfuzzer-options.py nss/fuzz/options/$name.options")
run-target "$fuzzer" "$name" "${options[@]}"
fi

update-status "cloning corpus for $name ($curr_target_n/$total_targets)"
clone-nssfuzz-corpus "$name"

update-status "running $name ($curr_target_n/$total_targets)"
run-nssfuzz-target "$target" "$name"
((curr_target_n++))
done

# Generate cryptofuzz headers
Expand Down Expand Up @@ -224,8 +178,13 @@ popd

# Clone cryptofuzz nss corpus
update-status "cloning cryptofuzz nss corpus"
clone-corpus "cryptofuzz" \
"https://storage.googleapis.com/cryptofuzz-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/cryptofuzz_cryptofuzz-nss/public.zip"
mkdir -p ./corpus/cryptofuzz

pushd ./corpus/cryptofuzz
retry-curl -O "https://storage.googleapis.com/cryptofuzz-backup.clusterfuzz-external.appspot.com/corpus/libFuzzer/cryptofuzz_cryptofuzz-nss/public.zip"
unzip public.zip
rm -f public.zip
popd

# Run cryptofuzz
update-status "running cryptofuzz"
Expand Down
19 changes: 0 additions & 19 deletions services/nss-coverage/libfuzzer-options.py

This file was deleted.

1 change: 1 addition & 0 deletions services/nss-coverage/setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ EDIT=1 SRCDIR=/src/fuzzing-tc ./fuzzing_tc.sh
./fuzzmanager.sh
./grcov.sh
./taskcluster.sh
./gsutil.sh

packages=(
binutils
Expand Down
Loading