test.yml

name: Teraslice CI Tests
# this will technically run again on merge to master, should limit it
on:
  pull_request:
    branches: [ master ]
env:
  NODE_VERSIONS: '[18, 22]'
  NODE_VERSION_MAIN: '22'
  NODE_VERSIONS_K8S: '[18, 22]'
  NODE_VERSIONS_EXT_STORAGE: '[22]'

jobs:
  compute-node-version-vars:
    runs-on: ubuntu-latest
    outputs:
      NODE_VERSIONS: ${{ steps.step1.outputs.NODE_VERSIONS }}
      NODE_VERSION_MAIN: ${{ steps.step2.outputs.NODE_VERSION_MAIN }}
      NODE_VERSIONS_EXT_K8S: ${{ steps.step3.outputs.NODE_VERSIONS_K8S }}
      NODE_VERSIONS_EXT_STORAGE: ${{ steps.step4.outputs.NODE_VERSIONS_EXT_STORAGE }}
    steps:
      - id: step1
        run: echo "NODE_VERSIONS=${{ env.NODE_VERSIONS }}" >> $GITHUB_OUTPUT
      - id: step2
        run: echo "NODE_VERSION_MAIN=${{ env.NODE_VERSION_MAIN }}" >> $GITHUB_OUTPUT
      - id: step3
        run: echo "NODE_VERSIONS_K8S=${{ env.NODE_VERSIONS_K8S }}" >> $GITHUB_OUTPUT
      - id: step4
        run: echo "NODE_VERSIONS_EXT_STORAGE=${{ env.NODE_VERSIONS_EXT_STORAGE }}" >> $GITHUB_OUTPUT

  verify-build:
    runs-on: ubuntu-latest
    needs: compute-node-version-vars
    steps:
    - name: Check out code
      uses: actions/checkout@v4

    - name: Setup Node
      uses: actions/setup-node@v4
      with:
        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSION_MAIN) }}
        # cache: 'yarn'

    - name: Install Yarn 4
      run: |
        corepack enable
        corepack prepare yarn@4.6.0 --activate
        yarn install

    - name: Install and build packages
      run: yarn setup

  check-docker-limit-before:
   uses: terascope/workflows/.github/workflows/check-docker-limit.yml@031a4b1777313a5f97820e4131ea6ac2396e5cfb
   secrets: inherit

  cache-docker-images:
    needs: check-docker-limit-before
    uses: terascope/workflows/.github/workflows/cache-docker-images.yml@031a4b1777313a5f97820e4131ea6ac2396e5cfb
    secrets: inherit

  linux-unit-tests:
    runs-on: ubuntu-latest
    needs: compute-node-version-vars
    strategy:
      matrix:
        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Setup Node ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
          # cache: 'yarn'

      - name: Install Yarn 4
        run: |
          corepack enable
          corepack prepare yarn@4.6.0 --activate
          yarn install

      - name: Install and build packages
        run: yarn setup

      - name: Test
        run: yarn test --suite unit --

  teraslice-elasticsearch-tests:
    runs-on: ubuntu-latest
    needs: [compute-node-version-vars, verify-build, cache-docker-images]
    strategy:
      # opensearch is finiky, keep testing others if it fails
      fail-fast: false
      matrix:
        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
        search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Setup Node ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
          # cache: 'yarn'

      - name: Install Yarn 4
        run: |
          corepack enable
          corepack prepare yarn@4.6.0 --activate
          yarn install

      # we login to docker to avoid docker pull limit rates
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Install and build packages
        run: yarn setup

      - name: Create Docker Image List
        run: |
          yarn docker:listImages
          cat ./images/image-list.txt

      - name: Restore Docker image cache
        id: docker-cache
        uses: actions/cache@v4
        with:
          path: /tmp/docker_cache
          key: docker-images-${{ hashFiles('./images/image-list.txt') }}

      - name: Test ${{ matrix.search-version }}
        run: yarn test:${{ matrix.search-version }}
        working-directory: ./packages/teraslice

  elasticsearch-store-tests:
    runs-on: ubuntu-latest
    needs: [compute-node-version-vars, verify-build, cache-docker-images]
    strategy:
      # opensearch is finiky, keep testing others if it fails
      fail-fast: false
      matrix:
        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
        search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Setup Node ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
          # cache: 'yarn'

      - name: Install Yarn 4
        run: |
          corepack enable
          corepack prepare yarn@4.6.0 --activate
          yarn install

      # we login to docker to avoid docker pull limit rates
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Install and build packages
        run: yarn setup

      - name: Create Docker Image List
        run: |
          yarn docker:listImages
          cat ./images/image-list.txt

      - name: Restore Docker image cache
        id: docker-cache
        uses: actions/cache@v4
        with:
          path: /tmp/docker_cache
          key: docker-images-${{ hashFiles('./images/image-list.txt') }}

      - name: Test ${{ matrix.search-version }}
        run: yarn test:${{ matrix.search-version }}
        working-directory: ./packages/elasticsearch-store

  lint-and-sync:
    runs-on: ubuntu-latest
    needs: [compute-node-version-vars, verify-build, cache-docker-images]
    # will remove the checkout, build and setup when the artifact is made to just
    # test the linting and syncing of the codebase
    steps:
    - name: Check out code
      uses: actions/checkout@v4

    - name: Setup Node
      uses: actions/setup-node@v4
      with:
        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSION_MAIN) }}
        # cache: 'yarn'

    - name: Install Yarn 4
      run: |
        corepack enable
        corepack prepare yarn@4.6.0 --activate
        yarn install

    - name: Install and build packages
      run: yarn setup

    - name: Lint codebase
      run: yarn lint

    - name: Sync codebase
      run: yarn sync --verify

  elasticsearch-api-tests:
    runs-on: ubuntu-latest
    needs: [compute-node-version-vars, verify-build, cache-docker-images]
    strategy:
      # opensearch is finiky, keep testing others if it fails
      fail-fast: false
      matrix:
        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
        search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Setup Node ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
          # cache: 'yarn'

      - name: Install Yarn 4
        run: |
          corepack enable
          corepack prepare yarn@4.6.0 --activate
          yarn install

      # we login to docker to avoid docker pull limit rates
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Install and build packages
        run: yarn setup

      - name: Create Docker Image List
        run: |
          yarn docker:listImages
          cat ./images/image-list.txt

      - name: Restore Docker image cache
        id: docker-cache
        uses: actions/cache@v4
        with:
          path: /tmp/docker_cache
          key: docker-images-${{ hashFiles('./images/image-list.txt') }}

      - name: Test ${{ matrix.search-version }}
        run: yarn test:${{ matrix.search-version }}
        working-directory: ./packages/elasticsearch-api

  e2e-tests:
    runs-on: ubuntu-latest
    needs: [compute-node-version-vars, verify-build, cache-docker-images]
    strategy:
      # opensearch is finiky, keep testing others if it fails
      fail-fast: false
      matrix:
        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
        search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Setup Node ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
          # cache: 'yarn'

      - name: Install Yarn 4
        run: |
          corepack enable
          corepack prepare yarn@4.6.0 --activate
          yarn install

      # we login to docker to avoid docker pull limit rates
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Install and build packages
        run: yarn setup

      - name: Create Docker Image List
        run: |
          yarn docker:listImages
          cat ./images/image-list.txt

      - name: Restore Docker image cache
        id: docker-cache
        uses: actions/cache@v4
        with:
          path: /tmp/docker_cache
          key: docker-images-${{ hashFiles('./images/image-list.txt') }}

      - name: Compile e2e code
        run: yarn build
        working-directory: ./e2e

      - name: Test ${{ matrix.search-version }}
        run: NODE_VERSION=${{ matrix.node-version }} yarn test:${{ matrix.search-version }}
        working-directory: ./e2e

  e2e-k8s-tests:
    runs-on: ubuntu-latest
    needs: [compute-node-version-vars, verify-build, cache-docker-images]
    strategy:
      # opensearch is finiky, keep testing others if it fails
      fail-fast: false
      matrix:
        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_K8S) }}
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Setup Node ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
          # cache: 'yarn'

      - name: Install Yarn 4
        run: |
          corepack enable
          corepack prepare yarn@4.6.0 --activate
          yarn install

      # we login to docker to avoid docker pull limit rates
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Install and build packages
        run: yarn setup

      - name: Create Docker Image List
        run: |
          yarn docker:listImages
          cat ./images/image-list.txt

      - name: Restore Docker image cache
        id: docker-cache
        uses: actions/cache@v4
        with:
          path: /tmp/docker_cache
          key: docker-images-${{ hashFiles('./images/image-list.txt') }}

      - name: Compile e2e code
        run: yarn build
        working-directory: ./e2e

      - name: Install Kind and Kubectl
        uses: helm/kind-action@v1.10.0
        with:
          install_only: "true"

      - name: Test k8s elasticsearch7
        run: NODE_VERSION=${{ matrix.node-version }} yarn test:k8s
        working-directory: ./e2e

  e2e-k8s-v2-tests:
    runs-on: ubuntu-latest
    needs: [compute-node-version-vars, verify-build, cache-docker-images]
    strategy:
      # opensearch is finiky, keep testing others if it fails
      fail-fast: false
      matrix:
        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_K8S) }}
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Setup Node ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
          # cache: 'yarn'

      - name: Install Yarn 4
        run: |
          corepack enable
          corepack prepare yarn@4.6.0 --activate
          yarn install

      # we login to docker to avoid docker pull limit rates
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Install and build packages
        run: yarn setup

      - name: Create Docker Image List
        run: |
          yarn docker:listImages
          cat ./images/image-list.txt

      - name: Restore Docker image cache
        id: docker-cache
        uses: actions/cache@v4
        with:
          path: /tmp/docker_cache
          key: docker-images-${{ hashFiles('./images/image-list.txt') }}

      - name: Compile e2e code
        run: yarn build
        working-directory: ./e2e

      - name: Install Kind and Kubectl
        uses: helm/kind-action@v1.10.0
        with:
          install_only: "true"

      - name: Test k8s V2 elasticsearch7
        run: NODE_VERSION=${{ matrix.node-version }} yarn test:k8sV2
        working-directory: ./e2e

  e2e-external-storage-tests:
    runs-on: ubuntu-latest
    needs: [compute-node-version-vars, verify-build, cache-docker-images]
    strategy:
      # opensearch is finiky, keep testing others if it fails
      fail-fast: false
      matrix:
        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_STORAGE) }}
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Setup Node ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
          # cache: 'yarn'

      - name: Install Yarn 4
        run: |
          corepack enable
          corepack prepare yarn@4.6.0 --activate
          yarn install

      # we login to docker to avoid docker pull limit rates
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Install and build packages
        run: yarn setup

      - name: Create Docker Image List
        run: |
          yarn docker:listImages
          cat ./images/image-list.txt

      - name: Restore Docker image cache
        id: docker-cache
        uses: actions/cache@v4
        with:
          path: /tmp/docker_cache
          key: docker-images-${{ hashFiles('./images/image-list.txt') }}

      - name: Compile e2e code
        run: yarn build
        working-directory: ./e2e

      - name: Test external Asset Storage opensearch1
        run: NODE_VERSION=${{ matrix.node-version }} yarn test:s3AssetStorage
        working-directory: ./e2e

  e2e-external-storage-tests-encrypted:
    runs-on: ubuntu-latest
    needs: [compute-node-version-vars, verify-build, cache-docker-images]
    strategy:
      # opensearch is finiky, keep testing others if it fails
      fail-fast: false
      matrix:
        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_STORAGE) }}
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Setup Node ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
          # cache: 'yarn'

      - name: Install Yarn 4
        run: |
          corepack enable
          corepack prepare yarn@4.6.0 --activate
          yarn install

      # we login to docker to avoid docker pull limit rates
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Install and build packages
        run: yarn setup

      - name: Create Docker Image List
        run: |
          yarn docker:listImages
          cat ./images/image-list.txt

      - name: Restore Docker image cache
        id: docker-cache
        uses: actions/cache@v4
        with:
          path: /tmp/docker_cache
          key: docker-images-${{ hashFiles('./images/image-list.txt') }}

      - name: Install mkcert
        run: curl -JLO "https://dl.filippo.io/mkcert/latest?for=linux/amd64" && sudo chmod 777 mkcert-v*-linux-amd64 && sudo cp mkcert-v*-linux-amd64 /usr/local/bin/mkcert

      - name: Install grep
        run: sudo apt update && sudo apt install grep

      - name: Check mkcert
        run: command -v mkcert

      - name: Check grep
        run: command -v grep

      - name: Compile e2e code
        run: yarn build
        working-directory: ./e2e

      - name: Test external Asset Storage opensearch1
        run: ENCRYPT_MINIO=true NODE_VERSION=${{ matrix.node-version }} yarn test:s3AssetStorage
        working-directory: ./e2e

  check-docker-limit-after:
    needs: [
      e2e-external-storage-tests-encrypted,
      e2e-external-storage-tests,
      e2e-k8s-v2-tests,
      e2e-k8s-tests,
      e2e-tests,
      elasticsearch-api-tests,
      elasticsearch-store-tests,
      teraslice-elasticsearch-tests
    ]
    uses: terascope/workflows/.github/workflows/check-docker-limit.yml@031a4b1777313a5f97820e4131ea6ac2396e5cfb
    secrets: inherit