Add resnet50 benchmark

benchmarks/resnet50/model_repository/dali/config.pbtxt

@@ -0,0 +1,52 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+name: "dali"
+backend: "dali"
+max_batch_size: 128
+input [
+  {
+    name: "DALI_INPUT_0"
+    data_type: TYPE_UINT8
+    dims: [ -1 ]
+    allow_ragged_batch: true
+  }
+]
+
+output [
+  {
+    name: "DALI_OUTPUT_0"
+    data_type: TYPE_FP32
+    dims: [ 224, 224, 3 ]
+  }
+]
+
+parameters: [
+  {
+    key: "num_threads"
+    value: { string_value: "4" }
+  }
+]
+
+dynamic_batching {
+  preferred_batch_size: [ 64, 128 ]
+  max_queue_delay_microseconds: 1
+}

benchmarks/resnet50/model_repository/dali/pipeline.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import nvidia.dali as dali
+import nvidia.dali.types as types
+
+
+def parse_args():
+    import argparse
+    parser = argparse.ArgumentParser(description="Serialize the pipeline and save it to a file")
+    parser.add_argument('file_path', type=str, help='The path where to save the serialized pipeline')
+    return parser.parse_args()
+
+
+def preprocessing(images, device='gpu'):
+  images = dali.fn.decoders.image(images, device="mixed" if device == 'gpu' else 'cpu', output_type=types.RGB)
+  images = dali.fn.resize(images, resize_x=224, resize_y=224)
+  return dali.fn.crop_mirror_normalize(images,
+                                       dtype=types.FLOAT,
+                                       output_layout="HWC",
+                                       crop=(224, 224),
+                                       mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
+                                       std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
+
+@dali.pipeline_def(batch_size=1, num_threads=1, device_id=0)
+def pipe():
+    images = dali.fn.external_source(device="cpu", name="DALI_INPUT_0", no_copy=True)
+    return preprocessing(images)
+
+
+def main(filename):
+    pipe().serialize(filename=filename)
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    main(args.file_path)
+

benchmarks/resnet50/model_repository/dali_trt_resnet50/config.pbtxt

@@ -0,0 +1,77 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+name: "dali_trt_resnet50"
+platform: "ensemble"
+max_batch_size: 128
+input [
+  {
+    name: "input"
+    data_type: TYPE_UINT8
+    dims: [ -1 ]
+    allow_ragged_batch: true
+  }
+]
+output [
+  {
+    name: "classes"
+    data_type: TYPE_INT32
+    dims: [ 1 ]
+  },
+  {
+    name: "probabilities"
+    data_type: TYPE_FP32
+    dims: [ 1001 ]
+  }
+]
+ensemble_scheduling {
+  step [
+    {
+      model_name: "dali"
+      model_version: -1
+      input_map {
+        key: "DALI_INPUT_0"
+        value: "input"
+      }
+      output_map {
+        key: "DALI_OUTPUT_0"
+        value: "preprocessed_image"
+      }
+    },
+    {
+      model_name: "resnet50_trt"
+      model_version: -1
+      input_map {
+        key: "input"
+        value: "preprocessed_image"
+      }
+      output_map {
+        key: "classes"
+        value: "classes"
+      }
+      output_map {
+        key: "probabilities"
+        value: "probabilities"
+      }
+    }
+  ]
+}

benchmarks/resnet50/model_repository/resnet50_onnx/config.pbtxt

@@ -0,0 +1,34 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+platform: "onnxruntime_onnx"
+backend: 'onnxruntime'
+max_batch_size: 128
+
+instance_group {
+  count: 2
+  kind: KIND_GPU
+}
+dynamic_batching {
+  preferred_batch_size: 64
+  preferred_batch_size: 128
+  max_queue_delay_microseconds: 1
+}

benchmarks/resnet50/model_repository/resnet50_tf/config.pbtxt

@@ -0,0 +1,43 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+name: "resnet50_tf"
+platform: "tensorflow_savedmodel"
+max_batch_size: 128
+
+# input [
+#   {
+#     name: "input_tensor:0"
+#     data_type: TYPE_FP32
+#     dims: [244, 244, 3]
+#   }
+# ]
+
+
+instance_group {
+  count: 2
+  kind: KIND_GPU
+}
+dynamic_batching {
+  preferred_batch_size: 64
+  preferred_batch_size: 128
+  max_queue_delay_microseconds: 1
+}

benchmarks/resnet50/model_repository/resnet50_trt/config.pbtxt

@@ -0,0 +1,29 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+platform: "tensorrt_plan"
+max_batch_size: 128
+
+dynamic_batching {
+  preferred_batch_size: 64
+  preferred_batch_size: 128
+  max_queue_delay_microseconds: 1
+}

benchmarks/resnet50/run-benchmarks.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# The MIT License (MIT)
+#
+# Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+pip install tqdm
+pip install scipy
+mkdir -p results
+
+echo "Benchmark dataset preprocessing"
+python scripts/dataset_preprocess.py --perf-file results/preprocessing.json
+
+echo "LOAD MODELS"
+python scripts/model-loader.py load -m dali
+python scripts/model-loader.py load -m resnet50_trt
+python scripts/model-loader.py load -m dali_trt_resnet50
+
+TIME_WINDOW=10000
+BATCH_SIZES="2 8 16 32 64 128"
+
+echo "WARM-UP"
+perf_analyzer -m dali_trt_resnet50 --input-data imagenet64.json --concurrency-range=128 -p$TIME_WINDOW
+
+echo "NN Benchmarks: single-sample"
+perf_analyzer -m resnet50_trt -p$TIME_WINDOW --concurrency-range=16:128:16 > results/nn-single-sample.txt
+
+echo "NN Benchmarks: batched"
+for BS in $BATCH_SIZES ; do perf_analyzer -m resnet50_trt -p$TIME_WINDOW -b$BS > results/nn-bs$BS.txt ; done
+
+echo "Ensemble Benchmarks: single-sample"
+perf_analyzer -m dali_trt_resnet50 -p$TIME_WINDOW --input-data imagenet64.json --concurrency-range=16:128:16 > results/ensemble-single-sample.txt
+
+echo "Ensemble Benchmarks: batched"
+for BS in $BATCH_SIZES ; do perf_analyzer -m dali_trt_resnet50 -p$TIME_WINDOW --input-data inputs-data/ --shape input:`stat --printf="%s" inputs-data/input` -b$BS > results/ensemble-bs$BS.txt; done