bench.yaml

default: &default
  matrix:
    LANGUAGE_SETTINGS: &lang_settings
      - { LANGUAGE: "r", VERSION: "0.8.1-syncronize" }
      - { LANGUAGE: "r", VERSION: "main" }
      - { LANGUAGE: "py", VERSION: "1.12.1" }

    DEVICE_SETTINGS: &device_settings
      - { DEVICE: "cpu" }
      - { DEVICE: "cuda" }

    BATCH_SIZE_ITER: &batch_size
      - { BATCH_SIZE: 32, ITER: 2000 }
      - { BATCH_SIZE: 256, ITER: 1000 }
      - { BATCH_SIZE: 512, ITER: 1000 }
      - { BATCH_SIZE: 1024, ITER: 500 }
      - { BATCH_SIZE: 2048, ITER: 500 }

small: &small
  matrix:
    LANGUAGE_SETTINGS: *lang_settings
    DEVICE_SETTINGS: *device_settings
    BATCH_SIZE_ITER: &small_batch_size_iter
      - { BATCH_SIZE: 32, ITER: 500 }
      - { BATCH_SIZE: 256, ITER: 100 }
      - { BATCH_SIZE: 512, ITER: 50 }
      - { BATCH_SIZE: 1024, ITER: 20 }
      - { BATCH_SIZE: 2048, ITER: 10 }

mini: &mini
  matrix:
    LANGUAGE_SETTINGS: *lang_settings
    DEVICE_SETTINGS: *device_settings
    BATCH_SIZE_ITER: &mini_batch_size_iter
      - { BATCH_SIZE: 32, ITER: 100 }
      - { BATCH_SIZE: 256, ITER: 20 }
      - { BATCH_SIZE: 512, ITER: 5 }
      - { BATCH_SIZE: 1024, ITER: 5 }
      - { BATCH_SIZE: 2048, ITER: 5 }

repeats: 10

bench:
  00_torch_mm: *default
  01_dollar_mm: *default
  02_nn_module: *default
  03_backward: *default
  04_optim: *default
  05_dataloader: *small
  06_cats_vs_dogs: *mini
  07_mnist:
    matrix:
      LANGUAGE_SETTINGS:
        - { LANGUAGE: "r", VERSION: "0.8.1-syncronize" }
        - { LANGUAGE: "r", VERSION: "main", VECTORIZED_DS: "yes" }
        - { LANGUAGE: "r", VERSION: "main", VECTORIZED_DS: "no" }
        - { LANGUAGE: "py", VERSION: "1.12.1" }
      DEVICE_SETTINGS: *device_settings
      BATCH_SIZE_ITER: *small_batch_size_iter