From 6ec708c771f786c7224b5aeb75e4bd9821f35985 Mon Sep 17 00:00:00 2001 From: Nicholas Tindle Date: Tue, 11 Jun 2024 12:29:23 -0500 Subject: [PATCH] ci(server): Agent Server CI (#7193) --- .github/workflows/autogpt-server-ci.yml | 268 ++++++++++++++++++++++++ rnd/autogpt_server/pyproject.toml | 10 +- rnd/autogpt_server/setup.py | 20 +- 3 files changed, 294 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/autogpt-server-ci.yml diff --git a/.github/workflows/autogpt-server-ci.yml b/.github/workflows/autogpt-server-ci.yml new file mode 100644 index 000000000000..6eabaa132bf7 --- /dev/null +++ b/.github/workflows/autogpt-server-ci.yml @@ -0,0 +1,268 @@ +name: AutoGPT Server CI + +on: + push: + branches: [master, development, ci-test*] + paths: + - ".github/workflows/autogpt-server-ci.yml" + - "rnd/autogpt_server/**" + - "!autogpt/tests/vcr_cassettes" + pull_request: + branches: [master, development, release-*] + paths: + - ".github/workflows/autogpt-server-ci.yml" + - "rnd/autogpt_server/**" + - "!autogpt/tests/vcr_cassettes" + +concurrency: + group: ${{ format('autogpt-server-ci-{0}', github.head_ref && format('{0}-{1}', github.event_name, github.event.pull_request.number) || github.sha) }} + cancel-in-progress: ${{ startsWith(github.event_name, 'pull_request') }} + +defaults: + run: + shell: bash + working-directory: rnd/autogpt_server + +jobs: + test: + permissions: + contents: read + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + platform-os: [ubuntu, macos, macos-arm64, windows] + runs-on: ${{ matrix.platform-os != 'macos-arm64' && format('{0}-latest', matrix.platform-os) || 'macos-14' }} + + steps: + # Quite slow on macOS (2~4 minutes to set up Docker) + # - name: Set up Docker (macOS) + # if: runner.os == 'macOS' + # uses: crazy-max/ghaction-setup-docker@v3 + + - name: Start MinIO service (Linux) + if: runner.os == 'Linux' + working-directory: "." + run: | + docker pull minio/minio:edge-cicd + docker run -d -p 9000:9000 minio/minio:edge-cicd + + - name: Start MinIO service (macOS) + if: runner.os == 'macOS' + working-directory: ${{ runner.temp }} + run: | + brew install minio/stable/minio + mkdir data + minio server ./data & + + # No MinIO on Windows: + # - Windows doesn't support running Linux Docker containers + # - It doesn't seem possible to start background processes on Windows. They are + # killed after the step returns. + # See: https://github.com/actions/runner/issues/598#issuecomment-2011890429 + + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - id: get_date + name: Get date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Set up Python dependency cache + # On Windows, unpacking cached dependencies takes longer than just installing them + if: runner.os != 'Windows' + uses: actions/cache@v4 + with: + path: ${{ runner.os == 'macOS' && '~/Library/Caches/pypoetry' || '~/.cache/pypoetry' }} + key: poetry-${{ runner.os }}-${{ hashFiles('rnd/autogpt_server/poetry.lock') }} + + - name: Install Poetry (Unix) + if: runner.os != 'Windows' + run: | + curl -sSL https://install.python-poetry.org | python3 - + + if [ "${{ runner.os }}" = "macOS" ]; then + PATH="$HOME/.local/bin:$PATH" + echo "$HOME/.local/bin" >> $GITHUB_PATH + fi + + - name: Install Poetry (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + (Invoke-WebRequest -Uri https://install.python-poetry.org -UseBasicParsing).Content | python - + + $env:PATH += ";$env:APPDATA\Python\Scripts" + echo "$env:APPDATA\Python\Scripts" >> $env:GITHUB_PATH + + - name: Install Python dependencies + run: poetry install + + - name: Generate Prisma Client + run: poetry run prisma generate + + - name: Run Database Migrations + run: poetry run prisma migrate dev --name updates + + - name: Run pytest with coverage + run: | + poetry run pytest -vv \ + test + env: + CI: true + PLAIN_OUTPUT: True + + # - name: Upload coverage reports to Codecov + # uses: codecov/codecov-action@v4 + # with: + # token: ${{ secrets.CODECOV_TOKEN }} + # flags: autogpt-server,${{ runner.os }} + + build: + permissions: + contents: read + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + platform-os: [ubuntu, macos, macos-arm64, windows] + runs-on: ${{ matrix.platform-os != 'macos-arm64' && format('{0}-latest', matrix.platform-os) || 'macos-14' }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: true + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - id: get_date + name: Get date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Set up Python dependency cache + # On Windows, unpacking cached dependencies takes longer than just installing them + if: runner.os != 'Windows' + uses: actions/cache@v4 + with: + path: ${{ runner.os == 'macOS' && '~/Library/Caches/pypoetry' || '~/.cache/pypoetry' }} + key: poetry-${{ runner.os }}-${{ hashFiles('rnd/autogpt_server/poetry.lock') }} + + - name: Install Poetry (Unix) + if: runner.os != 'Windows' + run: | + curl -sSL https://install.python-poetry.org | python3 - + + if [ "${{ runner.os }}" = "macOS" ]; then + PATH="$HOME/.local/bin:$PATH" + echo "$HOME/.local/bin" >> $GITHUB_PATH + fi + + - name: Install Poetry (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + (Invoke-WebRequest -Uri https://install.python-poetry.org -UseBasicParsing).Content | python - + + $env:PATH += ";$env:APPDATA\Python\Scripts" + echo "$env:APPDATA\Python\Scripts" >> $env:GITHUB_PATH + + - name: Install Python dependencies + run: poetry install + + - name: Generate Prisma Client + run: poetry run prisma generate + + - name: Run Database Migrations + run: poetry run prisma migrate dev --name updates + + - name: install rpm + if: matrix.platform-os == 'ubuntu' + run: sudo apt-get install -y alien fakeroot rpm + + - name: Build distribution + run: | + case "${{ matrix.platform-os }}" in + "macos" | "macos-arm64") + ${MAC_COMMAND} + ;; + "windows") + ${WINDOWS_COMMAND} + ;; + *) + ${LINUX_COMMAND} + ;; + esac + env: + MAC_COMMAND: "poetry run poe dist_dmg" + WINDOWS_COMMAND: "poetry run poe dist_msi" + LINUX_COMMAND: "poetry run poe dist_appimage" + + # break this into seperate steps each with their own name that matches the file + - name: Upload App artifact + uses: actions/upload-artifact@v4 + with: + name: autogptserver-app-${{ matrix.platform-os }} + path: /Users/runner/work/AutoGPT/AutoGPT/rnd/autogpt_server/build/*.app + + - name: Upload dmg artifact + uses: actions/upload-artifact@v4 + with: + name: autogptserver-dmg-${{ matrix.platform-os }} + path: /Users/runner/work/AutoGPT/AutoGPT/rnd/autogpt_server/build/AutoGPTServer.dmg + + - name: Upload msi artifact + uses: actions/upload-artifact@v4 + with: + name: autogptserver-msi-${{ matrix.platform-os }} + path: D:\a\AutoGPT\AutoGPT\rnd\autogpt_server\dist\*.msi + + - name: Upload deb artifact + uses: actions/upload-artifact@v4 + with: + name: autogptserver-deb-${{ matrix.platform-os }} + path: /Users/runner/work/AutoGPT/AutoGPT/rnd/autogpt_server/build/*.deb + + - name: Upload rpm artifact + uses: actions/upload-artifact@v4 + with: + name: autogptserver-rpm-${{ matrix.platform-os }} + path: /Users/runner/work/AutoGPT/AutoGPT/rnd/autogpt_server/build/*.rpm + + - name: Upload tar.gz artifact + uses: actions/upload-artifact@v4 + with: + name: autogptserver-tar.gz-${{ matrix.platform-os }} + path: /Users/runner/work/AutoGPT/AutoGPT/rnd/autogpt_server/build/*.tar.gz + + - name: Upload zip artifact + uses: actions/upload-artifact@v4 + with: + name: autogptserver-zip-${{ matrix.platform-os }} + path: /Users/runner/work/AutoGPT/AutoGPT/rnd/autogpt_server/build/*.zip + + - name: Upload pkg artifact + uses: actions/upload-artifact@v4 + with: + name: autogptserver-pkg-${{ matrix.platform-os }} + path: /Users/runner/work/AutoGPT/AutoGPT/rnd/autogpt_server/build/*.pkg + + - name: Upload AppImage artifact + uses: actions/upload-artifact@v4 + with: + name: autogptserver-AppImage-${{ matrix.platform-os }} + path: /Users/runner/work/AutoGPT/AutoGPT/rnd/autogpt_server/build/*.AppImage diff --git a/rnd/autogpt_server/pyproject.toml b/rnd/autogpt_server/pyproject.toml index 9ce3c3a00962..01ae64eca6f5 100644 --- a/rnd/autogpt_server/pyproject.toml +++ b/rnd/autogpt_server/pyproject.toml @@ -53,8 +53,8 @@ install = ["build", "_dinstall"] [tool.poe.tasks._dbuild] cmd = "python setup.py build" -[tool.poe.tasks.dist_mac] -cmd = "python setup.py bdist_mac" +[tool.poe.tasks.dist_app] +cmd = "python setup.py bdist_app" [tool.poe.tasks.dist_dmg] cmd = "python setup.py bdist_dmg" @@ -62,6 +62,12 @@ cmd = "python setup.py bdist_dmg" [tool.poe.tasks.dist_msi] cmd = "python setup.py bdist_msi" +[tool.poe.tasks.dist_appimage] +cmd = "python setup.py bdist_appimage" + +[tool.poe.tasks.dist_deb] +cmd = "python setup.py bdist_deb" + [tool.poe.tasks._dinstall] cmd = "python setup.py install" diff --git a/rnd/autogpt_server/setup.py b/rnd/autogpt_server/setup.py index bb4b8cab2f62..37d05c25c6ab 100644 --- a/rnd/autogpt_server/setup.py +++ b/rnd/autogpt_server/setup.py @@ -20,6 +20,7 @@ setup( name="AutoGPT Server", url="https://agpt.co", + # The entry points of the application executables=[ Executable( "autogpt_server/app.py", @@ -28,10 +29,11 @@ icon=icon, ), Executable( - "autogpt_server/cli.py", target_name="agpt_cli", base="console", icon=icon + "autogpt_server/cli.py", target_name="agpt_server_cli", base="console", icon=icon ), ], options={ + # Options for building all the executables "build_exe": { "packages": packages, "includes": [ @@ -41,22 +43,36 @@ "uvicorn.protocols.websockets.auto", "uvicorn.lifespan.on", ], + # Exclude the two module from readability.compat as it causes issues "excludes": ["readability.compat.two"], }, + # Mac .app specific options "bdist_mac": { "bundle_name": "AutoGPT", "iconfile": "../../assets/gpt_dark_RGB.icns", # "include_resources": ["IMG_3775.jpeg"], }, + # Mac .dmg specific options "bdist_dmg": { "applications_shortcut": True, - "volume_label": "AutoGPT Server", + "volume_label": "AutoGPTServer", }, + # Windows .msi specific options "bdist_msi": { "target_name": "AutoGPTServer", "add_to_path": True, "install_icon": "../../assets/gpt_dark_RGB.ico", }, + # Linux .appimage specific options "bdist_appimage": {}, + # Linux rpm specific options + "bdist_rpm": { + "name": "AutoGPTServer", + "description": "AutoGPT Server", + "version": "0.1", + "license": "UNKNOWNORPROPRIETARY", + "url": "https://agpt.co", + "long_description": "AutoGPT Server", + }, }, )