pipeline-main.yaml

name: Pipeline Domain Data Ingestion

on:
  workflow_dispatch:
  schedule:
    - cron: "0 22 1 * *" # At 22:00 on day-of-month 1.

env:
  DEBUG: false

jobs:
  migrate:
    name: Run Database Migrations
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: superfly/flyctl-actions/setup-flyctl@master
      - run: flyctl proxy 5432:5432 -a checkstatusgovbr-db &> /dev/null &
        env:
          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: "3.10"
      - name: Execute alembic migrations
        run: |
          pip install -U pip
          pip install poetry
          poetry install
          poetry run alembic upgrade head
        working-directory: api
        env:
          POETRY_VIRTUALENVS_CREATE: false
          DATABASE_URL: ${{ secrets.DATABASE_URL }}

  ingestion:
    needs: migrate
    name: Run Data Ingestion
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: superfly/flyctl-actions/setup-flyctl@master
      - run: flyctl proxy 5432:5432 -a checkstatusgovbr-db &> /dev/null &
        env:
          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: "3.10"
      - name: Execute ingestion
        run: |
          pip install -U pip
          pip install poetry
          poetry install
          python src/main.py
        working-directory: pipeline
        env:
          POETRY_VIRTUALENVS_CREATE: false
          DATABASE_URL: ${{ secrets.DATABASE_URL }}
          PYTHONPATH: ../