test-all.yaml

name: Test all functions, build marketplace

on:
  push:
    branches:
      - development
      - master

jobs:
  build_strategy_matrix:
    runs-on: ubuntu-latest
    steps:
      - name: Get the current branch name
        shell: bash
        run: echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
        id: myref
 
      - uses: actions/checkout@v3
      - id: set-matrix
        # This is very hacky, but it goes like that:
        # 1) Associate base_ref with origin/base_ref since actions/checkout doesn't do it, if we don't do that we won't be able to check the actual diff
        # 2) Build JSON string
        #  2.1) Add beginning of JSON
        #  2.2) Get diff between origin/base_ref and the checked-out repo => git diff ${{ github.base_ref }} --name-only
        #  2.3) Clean the file name and leave us only with directories => sed 's,/*[^/]\+/*$,,'
        #  2.4) Sort and keep only unique directories => sort | uniq
        #  2.5) Remove directories starting with '.' => grep -v '^\.'
        #  2.6) Add quotation marks to all strings => sed 's/.*/"&"/'
        #  2.7) Add comma suffix to all strings excluding the last one => sed '$!s/.*/&,/'
        #  2.8) Close JSON
        # 3) Save matrix JSON to output
        # This is old fetch command it cant work cause base_ref is only avaliable on pull request actions: git fetch origin ${{ github.base_ref }}:${{ github.base_ref }}
        run: |
          git fetch --no-tags --prune --depth=1 origin +refs/heads/${{ steps.myref.outputs.branch }}:refs/remotes/origin/${{ steps.myref.outputs.branch }}
          matrix=$((
            echo '{ "package" : ['
            git ls-files *[^cli] | grep '/' | sort | uniq| sed 's,/*[^/]\+/*$,,' | sort | uniq | grep -v '^\.' | sed 's/.*/"&"/' | sed '$!s/.*/&,/'
            echo " ]}"
          ) | jq -c .)
          echo "::set-output name=matrix::$matrix"
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}

  check_matrix:
    runs-on: ubuntu-latest
    needs: build_strategy_matrix
    steps:
      - name: Install json2yaml
        run: |
          sudo npm install -g json2yaml
      - name: Check matrix definition
        run: |
          matrix='${{ needs.build_strategy_matrix.outputs.matrix }}'
          echo $matrix
          echo $matrix | jq .
          echo $matrix | json2yaml
  run_monorepo_tests:
    needs: build_strategy_matrix
    runs-on: ubuntu-latest
    strategy:
      # matrix: [{"package": some package that changed}, {...}, ...]
      matrix: ${{fromJson(needs.build_strategy_matrix.outputs.matrix)}}
    steps:
      # Source
      - name: Checkout current repo
        uses: actions/checkout@v3
        with:
          path: functions
      # Install python 3.9
      - name: Install python 3.9
        uses: actions/setup-python@v3
        with:
          python-version: 3.9
      # Install dependencies
      - uses: actions/cache@v3
        id: cache
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('functions/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: Install requirements
        run: |
          pip install --upgrade pip
          pip install -r functions/requirements.txt

      - name: Run py tests
        run: python functions/functions.py  run-tests -r functions -s py -fn ${{ matrix.package }}
        continue-on-error: true
#      - name: Run ipynb tests
#        run: python functions/functions.py  run-tests -r functions -s ipynb
      - name: Clean
        run: |
          rm -rf functions

  build-marketplace:
    name: Build marketplace
    if: github.repository == 'mlrun/functions' && github.event_name != 'pull_request'
    runs-on: ubuntu-latest
    needs: run_monorepo_tests
    continue-on-error: false
    
    steps:
      - name: Get the current branch name
        shell: bash
        run: echo "::set-output name=branch::${GITHUB_REF#refs/heads/}"
        id: branch
      - name: Checkout current repo
        uses: actions/checkout@v3
        with:
          path: functions
      - name: Checkout Marketplace
        uses: actions/checkout@v3
        with:
          repository: mlrun/marketplace
          path: marketplace
      - name: Install python 3.9
        uses: actions/setup-python@v3
        with:
          python-version: 3.9
      - name: Install requirements
        run: |
          cd functions
          pip install --upgrade pip
          pip install -r requirements.txt
      - name: Build marketplace docs
        env:
          CHANNEL: ${{ steps.branch.outputs.branch }}
        run: |
          cd marketplace
          pwd
          git pull origin
          cd ..
          python functions/functions.py build-marketplace -s functions -m marketplace -c $CHANNEL -v -f
      - name: Publish marketplace release
        env:
          GITHUB_TOKEN: ${{ secrets.MARKETPLACE_ACCESS_TOKEN_V3 }}
          USERNAME: iguazio-cicd
          REPO_PATH: marketplace
          BASE_REPO: mlrun
          BASE_BRANCH: master
        run: |
          cd marketplace
          pwd
          COMMIT_SHA=$(git rev-parse --short "$GITHUB_SHA")
          echo "commit sha: $COMMIT_SHA"
          echo "github sha: $GITHUB_SHA"
          BRANCH_NAME=marketplace-doc-gen-$COMMIT_SHA
          REMOTE=https://$USERNAME:$GITHUB_TOKEN@github.com/$BASE_REPO/$REPO_PATH.git
          echo "Validating environment params...";
          [ -z "${GITHUB_TOKEN}" ] && {
              echo 'Missing input "GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}".';
              exit 1;
          };
          git config --local user.name $USERNAME
          git branch --set-upstream-to origin/master
          git remote -v
          echo "Checking out [$BRANCH_NAME]..."
          git checkout -b $BRANCH_NAME
          echo "Checking out [$BASE_BRANCH]..."
          git checkout $BASE_BRANCH
          git pull
          echo "Checking out [$BRANCH_NAME]..."
          git checkout $BRANCH_NAME
          git merge $BASE_BRANCH
          git status
          git status --ignored
          find . -type f | xargs ls -artl
          git add --all 
          git status
          git status --ignored
          echo "Commiting changes..."
          git rev-parse --show-toplevel
          git commit -a -m "Automatically generated by github-worflow[bot] for commit: $COMMIT_SHA"
          git status
          git status --ignored
          # https://stackoverflow.com/questions/64270867/auth-error-trying-to-copy-a-repo-with-github-actions
          git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | xargs -L1 git config --unset-all
          echo "Pushing [$BRANCH_NAME] to remote [$REMOTE]"
          git push -f $REMOTE $BRANCH_NAME
          echo "Submiting pull request..."
          gh pr create --title "Marketplace update from $BRANCH_NAME" --body "github-workflow" --base $BASE_BRANCH --head $BRANCH_NAME --repo $BASE_REPO/$REPO_PATH