Merge pull request #36 from hit9/dev

Add readme for benchmark and add one test more
hit9 · Feb 1, 2021 · 8f751d6 · 8f751d6
2 parents e7e1814 + ec37c90
commit 8f751d6
Show file tree

Hide file tree

Showing 13 changed files with 567 additions and 4 deletions.
diff --git a/benchmark/stm32/README.rst b/benchmark/stm32/README.rst
@@ -32,11 +32,11 @@ the optimization mode flag ``-O``, that is to use `the bitproto c lib <../../lib
      - 10000
      - 170μs
      - 163μs
-   * - ``-O2``
+   * - ``gcc -O2``
      - 10000
      - 135μs
      - 131μs
-   * - ``-O3``
+   * - ``gcc -O3``
      - 10000
      - 127μs
      - 119μs
@@ -58,7 +58,7 @@ the optimization mode flag ``-O``, that is not to use `the bitproto c lib <../..
      - 10000
      - 9μs
      - 9μs
-   * - ``-O2``
+   * - ``gcc -O2``
      - 10000
      - 9μs
      - 9μs
diff --git a/benchmark/unix/Makefile b/benchmark/unix/Makefile
@@ -27,7 +27,7 @@ bench-standard: bp
 
 bench-c-o1: bp
 	@echo "===================================================="
-	@echo "                Benchmark C (gcc -O2)               "
+	@echo "                Benchmark C (gcc -O1)               "
 	@echo "===================================================="
 	make -C C CC_OPTIMIZE=-O1
 

diff --git a/benchmark/unix/README.rst b/benchmark/unix/README.rst
@@ -4,6 +4,266 @@ bitproto benchmark on Unix OS
 This directory contains a simple benchmark for bitproto on Unix-like system (linux, macos etc).
 
 
+Benchmark result
+----------------
+
+Github Actions
+^^^^^^^^^^^^^^
+
+The results of bitproto benchmark on unix like system can be found on `github actions <https://github.com/hit9/bitproto/actions?query=workflow%3A%22bitproto+benchmark%22>`_,
+which are runned on `virtual machines <https://docs.github.com/en/actions/reference/specifications-for-github-hosted-runners#supported-runners-and-hardware-resources>`_
+(with 2-core CPU).
+
+The data in the following tables comes from `this github actions run <https://github.com/hit9/bitproto/actions/runs/526600150>`_.
+
+Standard Mode
+''''''''''''''
+
+The "standard mode" means to run bitproto compiler **without**
+the optimization mode flag ``-O``, that is to use `the bitproto lib <../../lib>`_.
+
+
+.. list-table::
+   :header-rows: 1
+
+   * - OS
+     - Language
+     - Mode
+     - GCC Optimization Flag
+     - Number of calls
+     - Encode cost per call
+     - Decode cost per call
+   * - Ubuntu 20.04
+     - C
+     - Standard
+     - No ``-O``
+     - 1000000
+     - 1.63μs
+     - 1.55μs
+   * - Ubuntu 20.04
+     - C
+     - Standard
+     - ``gcc -O1``
+     - 1000000
+     - 0.78μs
+     - 0.72μs
+   * - Ubuntu 20.04
+     - C
+     - Standard
+     - ``gcc -O2``
+     - 1000000
+     - 0.57μs
+     - 0.59μs
+   * - Ubuntu 20.04
+     - Go
+     - Standard
+     - /
+     - 1000000
+     - 6.95μs
+     - 7.08μs
+   * - Ubuntu 20.04
+     - Python
+     - Standard
+     - /
+     - 10000
+     - 386μs
+     - 410μs
+   * - Ubuntu 18.04
+     - C
+     - Standard
+     - No ``-O``
+     - 1000000
+     - 1.96μs
+     - 1.91μs
+   * - Ubuntu 18.04
+     - C
+     - Standard
+     - ``gcc -O1``
+     - 1000000
+     - 0.87μs
+     - 0.82μs
+   * - Ubuntu 18.04
+     - C
+     - Standard
+     - ``gcc -O2``
+     - 1000000
+     - 0.66μs
+     - 0.68μs
+   * - Ubuntu 18.04
+     - Go
+     - Standard
+     - /
+     - 1000000
+     - 8.53μs
+     - 8.58μs
+   * - Ubuntu 18.04
+     - Python
+     - Standard
+     - /
+     - 10000
+     - 477μs
+     - 496μs
+   * - MacOS 10.15
+     - C
+     - Standard
+     - No ``-O``
+     - 1000000
+     - 1.77μs
+     - 1.76μs
+   * - MacOS 10.15
+     - C
+     - Standard
+     - ``gcc -O1``
+     - 1000000
+     - 0.79μs
+     - 0.76μs
+   * - MacOS 10.15
+     - C
+     - Standard
+     - ``gcc -O1``
+     - 1000000
+     - 0.48μs
+     - 0.15μs
+   * - MacOS 10.15
+     - Go
+     - Standard
+     - /
+     - 1000000
+     - 7.66μs
+     - 7.70μs
+   * - MacOS 10.15
+     - Python
+     - Standard
+     - /
+     - 10000
+     - 553μs
+     - 563μs
+
+Optimization Mode
+''''''''''''''''''
+
+The "optimization mode" means to run bitproto compiler **with**
+the optimization mode flag ``-O``, that is not to use `the bitproto lib <../../lib>`_.
+
+.. list-table::
+   :header-rows: 1
+
+   * - OS
+     - Language
+     - Mode
+     - GCC Optimization Flag
+     - Number of calls
+     - Encode cost per call
+     - Decode cost per call
+   * - Ubuntu 20.04
+     - C
+     - ``bitproto -O``
+     - No ``-O``
+     - 1000000
+     - 0.16μs
+     - 0.17μs
+   * - Ubuntu 20.04
+     - C
+     - ``bitproto -O``
+     - ``gcc -O2``
+     - 1000000
+     - 0.06μs
+     - 0.06μs
+   * - Ubuntu 20.04
+     - Go
+     - ``bitproto -O``
+     - /
+     - 1000000
+     - 0.12μs
+     - 0.08μs
+   * - MacOS 10.15
+     - C
+     - ``bitproto -O``
+     - No ``-O``
+     - 1000000
+     - 0.16μs
+     - 0.15μs
+   * - MacOS 10.15
+     - C
+     - ``bitproto -O``
+     - ``gcc -O2``
+     - 1000000
+     - 0.06μs
+     - 0.06μs
+   * - MacOS 10.15
+     - Go
+     - ``bitproto -O``
+     - /
+     - 1000000
+     - 0.14μs
+     - 0.08μs
+
+Macbook
+''''''''
+
+The following table is the benchmark result on my macbook (Intel Core i5, 2 CPU Core, 2133 MHz).
+
+.. list-table::
+   :header-rows: 1
+   * - Language
+     - Mode
+     - GCC Optimization Flag
+     - Number of calls
+     - Encode cost per call
+     - Decode cost per call
+   * - C
+     - Standard
+     - No ``-O``
+     - 1000000
+     - 1.46μs
+     - 1.40μs
+   * - C
+     - Standard
+     - ``gcc -O1``
+     - 1000000
+     - 0.66μs
+     - 1.64μs
+   * - C
+     - Standard
+     - ``gcc -O2``
+     - 1000000
+     - 0.44μs
+     - 0.47μs
+   * - Go
+     - Standard
+     - /
+     - 1000000
+     - 6.57μs
+     - 6.71μs
+   * - Python
+     - Standard
+     - /
+     - 10000
+     - 492μs
+     - 495μs
+   * - C
+     - ``bitproto -O``
+     - No ``-O``
+     - 1000000
+     - 0.16μs
+     - 0.16μs
+   * - C
+     - ``bitproto -O``
+     - ``gcc -O2``
+     - 1000000
+     - 0.07μs
+     - 0.06μs
+   * - Go
+     - ``bitproto -O``
+     - /
+     - 1000000
+     - 0.14μs
+     - 0.07μs
+
+
+How to reproduce
+-----------------
+
 * Run benchmark for C / Go / Python:
 
   .. sourcecode:: bash

diff --git a/tests/test_encoding/encoding-cases/scatter/Makefile b/tests/test_encoding/encoding-cases/scatter/Makefile
@@ -0,0 +1,52 @@
+NAME=scatter
+BIN=main
+
+BP_FILENAME=$(NAME).bitproto
+BP_C_FILENAME=$(NAME)_bp.c
+BP_GO_FILENAME=$(NAME)_bp.go
+BP_PY_FILENAME=$(NAME)_bp.py
+BP_LIB_DIR=../../../../../lib/c
+BP_LIC_C_PATH=$(BP_LIB_DIR)/bitproto.c
+
+C_SOURCE_FILE=main.c
+C_SOURCE_FILE_LIST=$(C_SOURCE_FILE) $(BP_C_FILENAME) $(BP_LIC_C_PATH) 
+C_BIN=$(BIN)
+
+GO_BIN=$(BIN)
+
+PY_SOURCE_FILE=main.py
+
+CC_OPTIMIZATION_ARG?=
+
+OPTIMIZATION_MODE_ARGS?=
+
+bp-c:
+	@bitproto c $(BP_FILENAME) c/ $(OPTIMIZATION_MODE_ARGS)
+
+bp-go:
+	@bitproto go $(BP_FILENAME) go/bp/ $(OPTIMIZATION_MODE_ARGS)
+
+bp-py:
+	@bitproto py $(BP_FILENAME) py/
+
+build-c: bp-c
+	@cd c && $(CC) $(C_SOURCE_FILE_LIST) -I. -I$(BP_LIB_DIR) -o $(C_BIN) $(CC_OPTIMIZATION_ARG)
+
+build-go: bp-go
+	@cd go && go build -o $(GO_BIN)
+
+build-py: bp-py
+
+run-c: build-c
+	@cd c && ./$(C_BIN)
+
+run-go: build-go
+	@cd go && ./$(GO_BIN)
+
+run-py: build-py
+	@cd py && python $(PY_SOURCE_FILE)
+
+clean:
+	@rm -fr c/$(C_BIN) go/$(GO_BIN) go/vendor */*_bp.* */**/*_bp.* py/__pycache__
+
+run: run-c run-go run-py
diff --git a/tests/test_encoding/encoding-cases/scatter/c/.clang-format b/tests/test_encoding/encoding-cases/scatter/c/.clang-format
@@ -0,0 +1,4 @@
+---
+BasedOnStyle: Google
+IndentWidth: 4
+---