From cda21b7bd78019919d3036be5a1210b7420452cb Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 5 Aug 2021 15:27:35 -0700 Subject: [PATCH 001/181] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 8b47ace..b190d91 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,7 @@ # static-ffmpeg-binaries Static binaries of FFmpeg, for multiple OS & CPU combinations, built from source in a GitHub Actions workflow. + +The GitHub Actions workflows in this repo are what is covered by the Apache license. + +The resulting FFmpeg binaries are built using GPL libraries, and are therefore published under the GPL license. +Please see the [releases page](https://github.com/joeyparrish/static-ffmpeg-binaries/releases/) for binaries. From 48442f157e38c91643dea6e37c83f6f36527dac8 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 5 Aug 2021 15:54:29 -0700 Subject: [PATCH 002/181] First attempt at a release workflow This will be tested, debugged, and developed right on GitHub Actions, so you will get to see every gory, broken commit along the way. --- .github/workflows/release.yaml | 174 +++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..9537928 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,174 @@ +name: Release + +# Runs when a new tag is created. Creates a release for that tag, then builds +# ffmpeg and ffprobe on all OS & CPU combinations, then attaches them to the +# release. +#TODO: run on tag, not every push +on: push +#on: +# push: +# tags + +env: + FFMPEG_TAG: n4.4 + LIBVPX_TAG: v1.9.0 + AOM_TAG: v3.1.2 + X264_TAG: stable + X265_TAG: stable + LAME_VERSION: 3.100 + OPUS_VERSION: 1.3.1 + VORBIS_VERSION: 1.3.7 + +jobs: + # FIXME: draft release +# draft_release: +# name: Draft release +# needs: setup +# runs-on: ubuntu-latest +# outputs: +# release_id: ${{ steps.draft_release.outputs.id }} +# steps: +# - name: Draft release +# id: draft_release +# uses: actions/create-release@v1 +# env: +# GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} +# with: +# tag_name: ${{ github.event.inputs.tag}} +# release_name: ${{ github.event.inputs.tag}} +# body: ${{ github.event.inputs.tag }} +# draft: true + + build: + # FIXME: depend on draft_release +# needs: draft_release + strategy: + matrix: + # TODO: Windows + # TODO: Mac + # TODO: arm64 + #os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] + os: ["ubuntu-latest"] + include: + - os: ubuntu-latest + os_name: linux + target_arch: x64 + exe_ext: "" + + name: Build ${{ matrix.os_name }} ${{ matrix.target_arch }} + runs-on: ${{ matrix.os }} + + steps: + - name: Install packages + if: runner.os == 'Linux' + run: | + sudo apt -y install \ + nasm \ + yasm \ + libffmpeg-nvenc-dev \ + libvdpau-dev + + - name: Install libvpx + run: | + git clone --depth 1 https://chromium.googlesource.com/webm/libvpx -b "$LIBVPX_TAG" + cd libvpx + ./configure \ + --enable-vp8 \ + --enable-vp9 \ + --enable-runtime-cpu-detect \ + --enable-static \ + --disable-shared + make + sudo make install + + - name: Install aom + run: | + git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" + mkdir aom_build + cd aom_build + cmake ../aom \ + -DENABLE_DOCS=OFF \ + -DENABLE_EXAMPLES=OFF \ + -DENABLE_TESTS=OFF \ + -DENABLE_TESTDATA=OFF \ + -DENABLE_TOOLS=OFF \ + -DCONFIG_RUNTIME_CPU_DETECT=1 \ + -DCONFIG_SHARED=0 + make + sudo make install + + - name: Install x264 + run: | + git clone --depth 1 https://code.videolan.org/videolan/x264.git -b "$X264_TAG" + cd x264 + ./configure \ + --enable-static + make + sudo make install + + - name: Install x265 + run: | + hg clone http://hg.videolan.org/x265 -r "$X265_TAG" + cd x265/build + cmake ../source + make + sudo make install + + - name: Install lame + run: | + curl -L -o lame-"$LAME_VERSION".tar.gz https://sourceforge.net/projects/lame/files/lame/"$LAME_VERSION"/lame-"$LAME_VERSION".tar.gz/download + tar xf lame-"$LAME_VERSION".tar.gz + cd lame-"$LAME_VERSION" + ./configure \ + --enable-static \ + --disable-shared + make + sudo make install + + - name: Install opus + run: | + curl -LO https://archive.mozilla.org/pub/opus/opus-"$OPUS_VERSION".tar.gz + tar xf opus-"$OPUS_VERSION".tar.gz + cd opus-"$OPUS_VERSION" + ./configure \ + --enable-static \ + --disable-shared + make + sudo make install + # The pkgconfig files for opus don't work in ffmpeg, at least on arm, without + # this edit to insist on linking libm after libopus. + sudo sed -e 's/-lopus/-lopus -lm/' -i /usr/local/lib/pkgconfig/opus.pc + + - name: Build ffmpeg and ffprobe + run: | + git clone --depth 1 https://git.ffmpeg.org/ffmpeg.git -b "$FFMPEG_TAG" + cd ffmpeg + # FIXME: not static binaries yet? + ./configure \ + --disable-ffplay \ + --enable-libvpx \ + --enable-libaom \ + --enable-libx264 \ + --enable-libx265 \ + --enable-libmp3lame \ + --enable-libopus \ + --enable-nvenc \ + --enable-vdpau \ + --enable-runtime-cpudetect \ + --enable-gpl \ + --enable-static + make + + # FIXME: attach binaries + # FIXME: publish +# publish_release: +# name: Publish release +# needs: build +# runs-on: ubuntu-latest +# steps: +# - name: Publish release +# uses: eregon/publish-release@v1 +# env: +# GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} +# with: +# release_id: ${{ needs.draft_release.outputs.release_id }} From 13aeeaed0bd583286e3a525d78ef3294e0a8b0f5 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 5 Aug 2021 19:59:37 -0700 Subject: [PATCH 003/181] Enable draft_release, attach binaries to release, and fix static build The fixes for the static build are working for me locally, but still need testing in GitHub's environment. The artifacts part has also not been tested yet. Not yet publishing the releases. That step will be last, so that at least we don't have a bunch of garbage releases on the repo. Drafts and tags can be more easily deleted. --- .github/workflows/release.yaml | 95 +++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 31 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9537928..a099f9d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -3,11 +3,9 @@ name: Release # Runs when a new tag is created. Creates a release for that tag, then builds # ffmpeg and ffprobe on all OS & CPU combinations, then attaches them to the # release. -#TODO: run on tag, not every push -on: push -#on: -# push: -# tags +on: + push: + tags env: FFMPEG_TAG: n4.4 @@ -20,28 +18,25 @@ env: VORBIS_VERSION: 1.3.7 jobs: - # FIXME: draft release -# draft_release: -# name: Draft release -# needs: setup -# runs-on: ubuntu-latest -# outputs: -# release_id: ${{ steps.draft_release.outputs.id }} -# steps: -# - name: Draft release -# id: draft_release -# uses: actions/create-release@v1 -# env: -# GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} -# with: -# tag_name: ${{ github.event.inputs.tag}} -# release_name: ${{ github.event.inputs.tag}} -# body: ${{ github.event.inputs.tag }} -# draft: true + draft_release: + name: Draft release + runs-on: ubuntu-latest + outputs: + release_id: ${{ steps.draft_release.outputs.id }} + steps: + - name: Draft release + id: draft_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} + with: + tag_name: ${{ github.event.inputs.tag }} + release_name: ${{ github.event.inputs.tag }} + body: ${{ github.event.inputs.tag }} + draft: true build: - # FIXME: depend on draft_release -# needs: draft_release + needs: draft_release strategy: matrix: # TODO: Windows @@ -61,6 +56,7 @@ jobs: steps: - name: Install packages if: runner.os == 'Linux' + shell: bash run: | sudo apt -y install \ nasm \ @@ -69,6 +65,7 @@ jobs: libvdpau-dev - name: Install libvpx + shell: bash run: | git clone --depth 1 https://chromium.googlesource.com/webm/libvpx -b "$LIBVPX_TAG" cd libvpx @@ -82,6 +79,7 @@ jobs: sudo make install - name: Install aom + shell: bash run: | git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" mkdir aom_build @@ -98,6 +96,7 @@ jobs: sudo make install - name: Install x264 + shell: bash run: | git clone --depth 1 https://code.videolan.org/videolan/x264.git -b "$X264_TAG" cd x264 @@ -107,17 +106,23 @@ jobs: sudo make install - name: Install x265 + shell: bash run: | hg clone http://hg.videolan.org/x265 -r "$X265_TAG" cd x265/build cmake ../source make sudo make install + # This adjustment to the x265 linker flags is needed, at least on + # arm, to successfully link against it statically. (-lgcc_s not + # found (or needed), and -lpthread missing) + sudo sed -e 's/-lgcc_s -lgcc -lgcc_s -lgcc/-lpthread -lgcc/' -i /usr/local/lib/pkgconfig/x265.pc - name: Install lame + shell: bash run: | curl -L -o lame-"$LAME_VERSION".tar.gz https://sourceforge.net/projects/lame/files/lame/"$LAME_VERSION"/lame-"$LAME_VERSION".tar.gz/download - tar xf lame-"$LAME_VERSION".tar.gz + tar xzf lame-"$LAME_VERSION".tar.gz cd lame-"$LAME_VERSION" ./configure \ --enable-static \ @@ -126,25 +131,29 @@ jobs: sudo make install - name: Install opus + shell: bash run: | curl -LO https://archive.mozilla.org/pub/opus/opus-"$OPUS_VERSION".tar.gz - tar xf opus-"$OPUS_VERSION".tar.gz + tar xzf opus-"$OPUS_VERSION".tar.gz cd opus-"$OPUS_VERSION" ./configure \ --enable-static \ --disable-shared make sudo make install - # The pkgconfig files for opus don't work in ffmpeg, at least on arm, without - # this edit to insist on linking libm after libopus. + # The pkgconfig linker flags for static opus don't work when ffmpeg + # checks for opus in configure. Linking libm after libopus fixes it. sudo sed -e 's/-lopus/-lopus -lm/' -i /usr/local/lib/pkgconfig/opus.pc - name: Build ffmpeg and ffprobe + shell: bash run: | git clone --depth 1 https://git.ffmpeg.org/ffmpeg.git -b "$FFMPEG_TAG" cd ffmpeg - # FIXME: not static binaries yet? ./configure \ + --pkg-config-flags="--static" \ + --extra-cflags="-static" \ + --extra-ldflags="-static" \ --disable-ffplay \ --enable-libvpx \ --enable-libaom \ @@ -158,8 +167,32 @@ jobs: --enable-gpl \ --enable-static make + # Show that these are not dynamic executables. Fail if they are. + ldd ffmpeg && exit 1 + ldd ffprobe && exit 1 + + - name: Prepare artifacts + shell: bash + run: | + mkdir artifacts + SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}${{ matrix.exe_ext}}" + cp ffmpeg/ffmpeg artifacts/ffmpeg"$SUFFIX" + cp ffmpeg/ffprobe artifacts/ffprobe-"$SUFFIX" + # Show MD5 sums that can be verified by users later if they want to + # check for authenticity. + cd artifacts + md5sum * + # TODO: Can we push the md5s into the release description? + + - name: Attach artifacts to release + uses: dwenegar/upload-release-assets@v1 + env: + GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} + with: + release_id: ${{ needs.draft_release.outputs.release_id }} + assets_path: artifacts + - # FIXME: attach binaries # FIXME: publish # publish_release: # name: Publish release From e35aca2d993a3290617bf5463f61f3d4c440b5db Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 5 Aug 2021 20:02:47 -0700 Subject: [PATCH 004/181] Fix push trigger syntax Hopefully. --- .github/workflows/release.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a099f9d..81821d6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -5,7 +5,8 @@ name: Release # release. on: push: - tags + tags: + - "*" env: FFMPEG_TAG: n4.4 From 61d94a89103df03567495ab9d5a8732dc8850182 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 5 Aug 2021 20:07:40 -0700 Subject: [PATCH 005/181] Correct push event inputs I had copied and pasted the old ones from Shaka Packager, and clearly I was wrong. Not sure if this string is perfect, but we will soon find out. --- .github/workflows/release.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 81821d6..48e1c20 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -31,9 +31,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} with: - tag_name: ${{ github.event.inputs.tag }} - release_name: ${{ github.event.inputs.tag }} - body: ${{ github.event.inputs.tag }} + tag_name: ${{ github.ref }} + release_name: ${{ github.ref }} + body: ${{ github.ref }} draft: true build: From 2037c6e13659752ab7e92ca34ea89f7e8a90cca0 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 5 Aug 2021 20:13:05 -0700 Subject: [PATCH 006/181] Fix tag name Trying to create a draft with refs/tags/foo does not work, even though the step does not fail. This adds a setup job first that computes the correct tag so that other jobs can reference it. --- .github/workflows/release.yaml | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 48e1c20..b19b919 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -19,8 +19,26 @@ env: VORBIS_VERSION: 1.3.7 jobs: + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.compute_tag.outputs.tag }} + steps: + - name: Compute tag + id: compute_tag + # We would like to strip refs/tags/ off of the incoming ref and just + # use the tag name. Subsequent jobs can refer to the "tag" output of + # this job to determine the correct tag name. + run: | + # Strip refs/tags/ from the input to get the tag name, then store + # that in output. + echo "::set-output name=tag::${{ github.ref }}" \ + | sed -e 's@refs/tags/@@' + draft_release: name: Draft release + needs: setup runs-on: ubuntu-latest outputs: release_id: ${{ steps.draft_release.outputs.id }} @@ -31,9 +49,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} with: - tag_name: ${{ github.ref }} - release_name: ${{ github.ref }} - body: ${{ github.ref }} + tag_name: ${{ needs.setup.outputs.tag }} + release_name: ${{ needs.setup.outputs.tag }} + body: ${{ needs.setup.outputs.tag }} draft: true build: From 1512c17e99fdac5d38e4cbfc75818a8b4ea85952 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 5 Aug 2021 20:17:48 -0700 Subject: [PATCH 007/181] Mostly revert previous commit The simpler way actually did work. I was mistaken. --- .github/workflows/release.yaml | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b19b919..f365402 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -19,39 +19,22 @@ env: VORBIS_VERSION: 1.3.7 jobs: - setup: - name: Setup - runs-on: ubuntu-latest - outputs: - tag: ${{ steps.compute_tag.outputs.tag }} - steps: - - name: Compute tag - id: compute_tag - # We would like to strip refs/tags/ off of the incoming ref and just - # use the tag name. Subsequent jobs can refer to the "tag" output of - # this job to determine the correct tag name. - run: | - # Strip refs/tags/ from the input to get the tag name, then store - # that in output. - echo "::set-output name=tag::${{ github.ref }}" \ - | sed -e 's@refs/tags/@@' - draft_release: name: Draft release - needs: setup runs-on: ubuntu-latest outputs: release_id: ${{ steps.draft_release.outputs.id }} steps: - name: Draft release id: draft_release + # TODO: See if we can do this ourselves with the API. uses: actions/create-release@v1 env: GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} with: - tag_name: ${{ needs.setup.outputs.tag }} - release_name: ${{ needs.setup.outputs.tag }} - body: ${{ needs.setup.outputs.tag }} + tag_name: ${{ github.ref }} + release_name: ${{ github.ref }} + body: "" draft: true build: @@ -202,8 +185,10 @@ jobs: cd artifacts md5sum * # TODO: Can we push the md5s into the release description? + # TODO: See if we can do this ourselves with the API. - name: Attach artifacts to release + # TODO: See if we can do this ourselves with the API. uses: dwenegar/upload-release-assets@v1 env: GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} @@ -219,6 +204,7 @@ jobs: # runs-on: ubuntu-latest # steps: # - name: Publish release +# # TODO: See if we can do this ourselves with the API. # uses: eregon/publish-release@v1 # env: # GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} From 312934c5f1400e46271ec168f5e0f2fad26bb20d Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 5 Aug 2021 21:03:14 -0700 Subject: [PATCH 008/181] Fix tag/version variables It turns out that 3.100 in yaml is a float, not a string. :-) This fixes all the version numbers and tags to be quoted strings, to avoid any future issues as versions change. --- .github/workflows/release.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f365402..97260b8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,14 +9,14 @@ on: - "*" env: - FFMPEG_TAG: n4.4 - LIBVPX_TAG: v1.9.0 - AOM_TAG: v3.1.2 - X264_TAG: stable - X265_TAG: stable - LAME_VERSION: 3.100 - OPUS_VERSION: 1.3.1 - VORBIS_VERSION: 1.3.7 + FFMPEG_TAG: "n4.4" + LIBVPX_TAG: "v1.9.0" + AOM_TAG: "v3.1.2" + X264_TAG: "stable" + X265_TAG: "stable" + LAME_VERSION: "3.100" + OPUS_VERSION: "1.3.1" + VORBIS_VERSION: "1.3.7" jobs: draft_release: From 80c0a465038b6d1592031a1c66bda44ea89281ef Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 5 Aug 2021 22:08:41 -0700 Subject: [PATCH 009/181] Add set -x to see the commands run This may help debug a failure at the end of the ffmpeg build steps. --- .github/workflows/release.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 97260b8..357f68f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -60,6 +60,7 @@ jobs: if: runner.os == 'Linux' shell: bash run: | + set -x sudo apt -y install \ nasm \ yasm \ @@ -69,6 +70,7 @@ jobs: - name: Install libvpx shell: bash run: | + set -x git clone --depth 1 https://chromium.googlesource.com/webm/libvpx -b "$LIBVPX_TAG" cd libvpx ./configure \ @@ -83,6 +85,7 @@ jobs: - name: Install aom shell: bash run: | + set -x git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" mkdir aom_build cd aom_build @@ -100,6 +103,7 @@ jobs: - name: Install x264 shell: bash run: | + set -x git clone --depth 1 https://code.videolan.org/videolan/x264.git -b "$X264_TAG" cd x264 ./configure \ @@ -110,6 +114,7 @@ jobs: - name: Install x265 shell: bash run: | + set -x hg clone http://hg.videolan.org/x265 -r "$X265_TAG" cd x265/build cmake ../source @@ -123,6 +128,7 @@ jobs: - name: Install lame shell: bash run: | + set -x curl -L -o lame-"$LAME_VERSION".tar.gz https://sourceforge.net/projects/lame/files/lame/"$LAME_VERSION"/lame-"$LAME_VERSION".tar.gz/download tar xzf lame-"$LAME_VERSION".tar.gz cd lame-"$LAME_VERSION" @@ -135,6 +141,7 @@ jobs: - name: Install opus shell: bash run: | + set -x curl -LO https://archive.mozilla.org/pub/opus/opus-"$OPUS_VERSION".tar.gz tar xzf opus-"$OPUS_VERSION".tar.gz cd opus-"$OPUS_VERSION" @@ -150,6 +157,7 @@ jobs: - name: Build ffmpeg and ffprobe shell: bash run: | + set -x git clone --depth 1 https://git.ffmpeg.org/ffmpeg.git -b "$FFMPEG_TAG" cd ffmpeg ./configure \ @@ -176,6 +184,7 @@ jobs: - name: Prepare artifacts shell: bash run: | + set -x mkdir artifacts SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}${{ matrix.exe_ext}}" cp ffmpeg/ffmpeg artifacts/ffmpeg"$SUFFIX" From 0890712c295e8e03882b1ebd84d2c7cf038f8ad4 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 09:15:04 -0700 Subject: [PATCH 010/181] Fix failing ffmpeg build script The last command, even if it doesn't run exit 1, returns a non-zero return code, causing the build script to fail. This adds "true" to the end to fix it. As with everything else in this repo, it's untested until I push to GitHub and try it in the Actions environment. --- .github/workflows/release.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 357f68f..c6db20c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -180,6 +180,9 @@ jobs: # Show that these are not dynamic executables. Fail if they are. ldd ffmpeg && exit 1 ldd ffprobe && exit 1 + # After commands that we expect to fail, we still need a successful + # command here to make this step a success. + true - name: Prepare artifacts shell: bash From 26589399fc97b312c78efcd1cdba672b799ff34c Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 09:17:14 -0700 Subject: [PATCH 011/181] Replace external action with python script If we can use the GitHub API directly, we will have more flexibility with regard to how we call it, and we will be able to do more, such as edit release notes later in the publication flow. --- .github/workflows/release.yaml | 35 +++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c6db20c..c8725c4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -27,15 +27,32 @@ jobs: steps: - name: Draft release id: draft_release - # TODO: See if we can do this ourselves with the API. - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} - with: - tag_name: ${{ github.ref }} - release_name: ${{ github.ref }} - body: "" - draft: true + shell: python + run: | + from __future__ import print_function + import base64 + import json + import os + import urllib.request + url = "https://api.github.com/repos/{}/releases".format( + os.environ["GITHUB_REPOSITORY"]) + # Turn "refs/tags/foo" into "foo". + tag_name = "${{ github.ref }}".split("/").pop() + data = { + "tag_name": tag_name, + "name": tag_name, + "draft": True, + } + auth_token = base64.encodestring("%s:%s" % ( + "shaka-bot", "${{ secrets.SHAKA_BOT_TOKEN }}")) + headers = { + "Authorization": "Basic {}".format(auth_token), + } + req = urllib.request.Request(url, data, headers) + with urllib.request.urlopen(req) as resp: + body = json.load(resp) + release_id = body["id"] + print("::set-output name=release_id::{}".format(release_id)) build: needs: draft_release From 5d418a5c5386c530a5d7d82a3992aec0cc549300 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 09:27:17 -0700 Subject: [PATCH 012/181] Second attempt at a python script for the GitHub API --- .github/workflows/release.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c8725c4..ba8c5dd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -29,7 +29,6 @@ jobs: id: draft_release shell: python run: | - from __future__ import print_function import base64 import json import os @@ -43,12 +42,14 @@ jobs: "name": tag_name, "draft": True, } - auth_token = base64.encodestring("%s:%s" % ( - "shaka-bot", "${{ secrets.SHAKA_BOT_TOKEN }}")) + login = "{}:{}".format("shaka-bot", "${{ secrets.SHAKA_BOT_TOKEN }}") + auth_token = base64.b64encode(login.encode("utf-8")).decode("utf-8") headers = { "Authorization": "Basic {}".format(auth_token), + "Content-Type": "application/json", } - req = urllib.request.Request(url, data, headers) + data_bytes = json.dumps(data).encode("utf-8") + req = urllib.request.Request(url, data_bytes, headers) with urllib.request.urlopen(req) as resp: body = json.load(resp) release_id = body["id"] From 9ec8a64d5f91988589730cfc8288eee440b3d413 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 09:43:49 -0700 Subject: [PATCH 013/181] Add a temporary debug step I want to know if I can refer to a local custom action from the workflow folder without checking out the repo explicitly. To this end, I am checking the current working directory, github workspace directory (should be the same), the path to the currently-running script, and listing the contents of the directory two levels up from the current one. The results of this will tell me what I can get away with, and to make it a quick test, I've added an explicit "exit 1" to make this step fail. --- .github/workflows/release.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ba8c5dd..19656d5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -25,6 +25,16 @@ jobs: outputs: release_id: ${{ steps.draft_release.outputs.id }} steps: + - name: Debug Workflow Path and PWD + shell: bash + run: | + echo "PWD = $(pwd)" + echo "GITHUB_WORKSPACE = $GITHUB_WORKSPACE" + echo "0 = $0" + echo "Contents, two levels up:" + find $(dirname $(dirname "$GITHUB_WORKSPACE")) + exit 1 + - name: Draft release id: draft_release shell: python From d3ef6f44eb551e34950d1111d90046d1da1cdd9f Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 10:34:45 -0700 Subject: [PATCH 014/181] Custom python action to replace third-party actions Not sure if this will work yet, as always. But this replaces all third-party actions with a custom action that cannot be poisoned with a supply-chain attack. It also adds additional features to the workflow, such as collecting the assets attached to the release and using their MD5 sums as the body of the release. --- .../custom-github-repo-api-action/action.yaml | 104 +++++++++++++++ .github/workflows/release.yaml | 119 ++++++++---------- 2 files changed, 157 insertions(+), 66 deletions(-) create mode 100644 .github/workflows/custom-github-repo-api-action/action.yaml diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml new file mode 100644 index 0000000..e407714 --- /dev/null +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -0,0 +1,104 @@ +name: GitHub Repo API Action + +description: | + A reusable action to talk to the GitHub API, wihtout relying on external + actions that could be affected by a supply-chain attack. + +inputs: + username: + description: A username for authentication to the GitHub API. + required: true + token: + description: A token for authentication to the GitHub API. + required: true + method: + description: A high-level method to invoke. See action source for details. + required: true + +runs: + using: composite + steps: + - name: Call Repo API + shell: python + run: | + import base64 + import json + import os + import urllib.request + + def call_api(path, data=None, headers=None, method=None, get_json=True): + repo = os.environ["GITHUB_REPOSITORY"] + url = "https://api.github.com/repos/{}{}".format(repo, path) + login = "${{ inputs.username }}:${{ inputs.token }}" + + if not headers: + headers = {} + if not method: + method = "POST" if data else "GET" + + auth_token = base64.b64encode(login.encode("utf-8")).decode("utf-8") + headers["Authorization"] = "Basic {}".format(auth_token) + + if type(data) is dict: + data_bytes = json.dumps(data).encode("utf-8") + headers["Content-Type"] = "application/json" + elif type(data) is str: + data_bytes = data.encode("utf-8") + elif not data: + data_bytes = None + else: + assert(type(data) is bytes) + data_bytes = data + + req = urllib.request.Request(url, data_bytes, headers) + with urllib.request.urlopen(req) as resp: + if get_json: + body = json.load(resp) + return body + else: + return resp.read() + + def draft_release(tag_name): + # Turns "refs/tags/foo" into "foo" + name = tag_name.split("/").pop() + + data = { + "tag_name": tag_name, + "name": name, + "draft": True, + } + body = call_api("/releases", data) + release_id = body["id"] + print("::set-output name=release_id::{}".format(release_id)) + + def upload_asset(release_id, asset_path, mime_type): + base_name = os.path.basename(asset_path) + path = "/releases/{}/assets?name={}".format(release_id, base_name) + with open(asset_path, "rb") as f: + data = f.read() + call_api(path, data, {"Content-Type": mime_type}) + + def upload_all_assets(release_id, folder): + for asset in os.listdir(folder): + asset_path = os.path.join(folder, asset) + mime_type = mimetypes.guess_type(asset_path) + upload_asset(release_id, asset_path, mime_type) + + def download_all_assets(release_id, output_path): + if not os.exists(output_path): + os.mkdir(output_path) + + path = "/releases/{}/assets".format(release_id) + asset_list = call_api(path) + for asset in asset_list: + name = asset["name"] + url = asset["browser_download_url"] + with urllib.request.urlopen(url) as download: + with open(os.path.join(output_path, name), "wb") as local_file: + local_file.write(download.read()) + + def update_release(release_id, **data): + path = "/releases/{}".format(release_id) + call_api(path, data, method="PATCH") + + ${{ inputs.method }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 19656d5..f872928 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -23,47 +23,19 @@ jobs: name: Draft release runs-on: ubuntu-latest outputs: - release_id: ${{ steps.draft_release.outputs.id }} + release_id: ${{ steps.draft_release.outputs.release_id }} steps: - - name: Debug Workflow Path and PWD + - name: Checkout repo shell: bash - run: | - echo "PWD = $(pwd)" - echo "GITHUB_WORKSPACE = $GITHUB_WORKSPACE" - echo "0 = $0" - echo "Contents, two levels up:" - find $(dirname $(dirname "$GITHUB_WORKSPACE")) - exit 1 + run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" -b "$GITHUB_REF" - name: Draft release id: draft_release - shell: python - run: | - import base64 - import json - import os - import urllib.request - url = "https://api.github.com/repos/{}/releases".format( - os.environ["GITHUB_REPOSITORY"]) - # Turn "refs/tags/foo" into "foo". - tag_name = "${{ github.ref }}".split("/").pop() - data = { - "tag_name": tag_name, - "name": tag_name, - "draft": True, - } - login = "{}:{}".format("shaka-bot", "${{ secrets.SHAKA_BOT_TOKEN }}") - auth_token = base64.b64encode(login.encode("utf-8")).decode("utf-8") - headers = { - "Authorization": "Basic {}".format(auth_token), - "Content-Type": "application/json", - } - data_bytes = json.dumps(data).encode("utf-8") - req = urllib.request.Request(url, data_bytes, headers) - with urllib.request.urlopen(req) as resp: - body = json.load(resp) - release_id = body["id"] - print("::set-output name=release_id::{}".format(release_id)) + uses: ./repo-src/.github/workflows/custom-github-repo-api-action + with: + username: shaka-bot + token: ${{ secrets.SHAKA_BOT_TOKEN }} + method: draft_release("${{ github.ref }}") build: needs: draft_release @@ -212,41 +184,56 @@ jobs: # command here to make this step a success. true - - name: Prepare artifacts + - name: Prepare assets shell: bash run: | set -x - mkdir artifacts + mkdir assets SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}${{ matrix.exe_ext}}" - cp ffmpeg/ffmpeg artifacts/ffmpeg"$SUFFIX" - cp ffmpeg/ffprobe artifacts/ffprobe-"$SUFFIX" + cp ffmpeg/ffmpeg assets/ffmpeg"$SUFFIX" + cp ffmpeg/ffprobe assets/ffprobe-"$SUFFIX" # Show MD5 sums that can be verified by users later if they want to # check for authenticity. - cd artifacts + cd assets md5sum * - # TODO: Can we push the md5s into the release description? - # TODO: See if we can do this ourselves with the API. - - - name: Attach artifacts to release - # TODO: See if we can do this ourselves with the API. - uses: dwenegar/upload-release-assets@v1 - env: - GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} + + - name: Checkout repo + shell: bash + run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" -b "$GITHUB_REF" + + - name: Attach assets to release + uses: ./repo-src/.github/workflows/custom-github-repo-api-action + with: + username: shaka-bot + token: ${{ secrets.SHAKA_BOT_TOKEN }} + method: upload_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/") + + publish_release: + name: Publish release + needs: build + runs-on: ubuntu-latest + steps: + - name: Get all release assets + uses: ./repo-src/.github/workflows/custom-github-repo-api-action + with: + username: shaka-bot + token: ${{ secrets.SHAKA_BOT_TOKEN }} + method: download_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/") + + - name: Checksum all release assets + id: checksum + shell: bash + run: | + cd assets + SUMS=$(md5sum *) + echo "::set-output name=sums::$SUMS" + # The sums are now in an output variable that we can use in the next + # step to set the release body. + + - name: Publish release + uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: - release_id: ${{ needs.draft_release.outputs.release_id }} - assets_path: artifacts - - - # FIXME: publish -# publish_release: -# name: Publish release -# needs: build -# runs-on: ubuntu-latest -# steps: -# - name: Publish release -# # TODO: See if we can do this ourselves with the API. -# uses: eregon/publish-release@v1 -# env: -# GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} -# with: -# release_id: ${{ needs.draft_release.outputs.release_id }} + username: shaka-bot + token: ${{ secrets.SHAKA_BOT_TOKEN }} + # TODO: publish for real, draft=False + method: update_release("${{ needs.draft_release.outputs.release_id }}", { body="${{ steps.checksum.outputs.sums }}", draft=True }) From 5470ed8197bd6062d9a3f6ea0d5759c3572d46cb Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 10:36:22 -0700 Subject: [PATCH 015/181] Add missing import to python API action --- .github/workflows/custom-github-repo-api-action/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index e407714..3198587 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -23,6 +23,7 @@ runs: run: | import base64 import json + import mimetypes import os import urllib.request From cd4dc347b55cb500ea530cedcbcedabd6e343cdb Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 10:37:39 -0700 Subject: [PATCH 016/181] Fix failing git clone on repo The -b option does not seem to work with refs/tags/foo --- .github/workflows/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f872928..7d4b29a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -27,7 +27,7 @@ jobs: steps: - name: Checkout repo shell: bash - run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" -b "$GITHUB_REF" + run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" - name: Draft release id: draft_release @@ -199,7 +199,7 @@ jobs: - name: Checkout repo shell: bash - run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" -b "$GITHUB_REF" + run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" - name: Attach assets to release uses: ./repo-src/.github/workflows/custom-github-repo-api-action From bac2424d40c45dac59cea76f97ac1ceba0821455 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 10:39:16 -0700 Subject: [PATCH 017/181] Fix git clone output dir Also adds a missing clone in the publish job --- .github/workflows/release.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7d4b29a..3743916 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -27,7 +27,7 @@ jobs: steps: - name: Checkout repo shell: bash - run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" + run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - name: Draft release id: draft_release @@ -199,7 +199,7 @@ jobs: - name: Checkout repo shell: bash - run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" + run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - name: Attach assets to release uses: ./repo-src/.github/workflows/custom-github-repo-api-action @@ -213,6 +213,10 @@ jobs: needs: build runs-on: ubuntu-latest steps: + - name: Checkout repo + shell: bash + run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + - name: Get all release assets uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: From c9dd0a248e5b79809eace1e5e0a89e8ec48e2309 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 15:09:40 -0700 Subject: [PATCH 018/181] Add a print to debug upload_assets failure --- .github/workflows/custom-github-repo-api-action/action.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index 3198587..d5eafde 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -36,6 +36,7 @@ runs: headers = {} if not method: method = "POST" if data else "GET" + print({'path': path, 'data': data, 'headers': headers, 'method': method}) auth_token = base64.b64encode(login.encode("utf-8")).decode("utf-8") headers["Authorization"] = "Basic {}".format(auth_token) @@ -77,7 +78,7 @@ runs: path = "/releases/{}/assets?name={}".format(release_id, base_name) with open(asset_path, "rb") as f: data = f.read() - call_api(path, data, {"Content-Type": mime_type}) + call_api(path, data, headers={"Content-Type": mime_type}) def upload_all_assets(release_id, folder): for asset in os.listdir(folder): From b4dac36a12a451504815350747a2cc36c6935db0 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 15:18:31 -0700 Subject: [PATCH 019/181] Skip build to debug upload process Instead of building ffmpeg over the course of 35 minutes, just try to upload the README.md file from ffmpeg as an asset, so we can debug the upload process more quickly. --- .github/workflows/release.yaml | 247 +++++++++++++++++---------------- 1 file changed, 124 insertions(+), 123 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3743916..74535b3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -56,133 +56,133 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Install packages - if: runner.os == 'Linux' - shell: bash - run: | - set -x - sudo apt -y install \ - nasm \ - yasm \ - libffmpeg-nvenc-dev \ - libvdpau-dev - - - name: Install libvpx - shell: bash - run: | - set -x - git clone --depth 1 https://chromium.googlesource.com/webm/libvpx -b "$LIBVPX_TAG" - cd libvpx - ./configure \ - --enable-vp8 \ - --enable-vp9 \ - --enable-runtime-cpu-detect \ - --enable-static \ - --disable-shared - make - sudo make install - - - name: Install aom - shell: bash - run: | - set -x - git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" - mkdir aom_build - cd aom_build - cmake ../aom \ - -DENABLE_DOCS=OFF \ - -DENABLE_EXAMPLES=OFF \ - -DENABLE_TESTS=OFF \ - -DENABLE_TESTDATA=OFF \ - -DENABLE_TOOLS=OFF \ - -DCONFIG_RUNTIME_CPU_DETECT=1 \ - -DCONFIG_SHARED=0 - make - sudo make install - - - name: Install x264 - shell: bash - run: | - set -x - git clone --depth 1 https://code.videolan.org/videolan/x264.git -b "$X264_TAG" - cd x264 - ./configure \ - --enable-static - make - sudo make install - - - name: Install x265 - shell: bash - run: | - set -x - hg clone http://hg.videolan.org/x265 -r "$X265_TAG" - cd x265/build - cmake ../source - make - sudo make install - # This adjustment to the x265 linker flags is needed, at least on - # arm, to successfully link against it statically. (-lgcc_s not - # found (or needed), and -lpthread missing) - sudo sed -e 's/-lgcc_s -lgcc -lgcc_s -lgcc/-lpthread -lgcc/' -i /usr/local/lib/pkgconfig/x265.pc - - - name: Install lame - shell: bash - run: | - set -x - curl -L -o lame-"$LAME_VERSION".tar.gz https://sourceforge.net/projects/lame/files/lame/"$LAME_VERSION"/lame-"$LAME_VERSION".tar.gz/download - tar xzf lame-"$LAME_VERSION".tar.gz - cd lame-"$LAME_VERSION" - ./configure \ - --enable-static \ - --disable-shared - make - sudo make install - - - name: Install opus - shell: bash - run: | - set -x - curl -LO https://archive.mozilla.org/pub/opus/opus-"$OPUS_VERSION".tar.gz - tar xzf opus-"$OPUS_VERSION".tar.gz - cd opus-"$OPUS_VERSION" - ./configure \ - --enable-static \ - --disable-shared - make - sudo make install - # The pkgconfig linker flags for static opus don't work when ffmpeg - # checks for opus in configure. Linking libm after libopus fixes it. - sudo sed -e 's/-lopus/-lopus -lm/' -i /usr/local/lib/pkgconfig/opus.pc +# - name: Install packages +# if: runner.os == 'Linux' +# shell: bash +# run: | +# set -x +# sudo apt -y install \ +# nasm \ +# yasm \ +# libffmpeg-nvenc-dev \ +# libvdpau-dev +# +# - name: Install libvpx +# shell: bash +# run: | +# set -x +# git clone --depth 1 https://chromium.googlesource.com/webm/libvpx -b "$LIBVPX_TAG" +# cd libvpx +# ./configure \ +# --enable-vp8 \ +# --enable-vp9 \ +# --enable-runtime-cpu-detect \ +# --enable-static \ +# --disable-shared +# make +# sudo make install +# +# - name: Install aom +# shell: bash +# run: | +# set -x +# git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" +# mkdir aom_build +# cd aom_build +# cmake ../aom \ +# -DENABLE_DOCS=OFF \ +# -DENABLE_EXAMPLES=OFF \ +# -DENABLE_TESTS=OFF \ +# -DENABLE_TESTDATA=OFF \ +# -DENABLE_TOOLS=OFF \ +# -DCONFIG_RUNTIME_CPU_DETECT=1 \ +# -DCONFIG_SHARED=0 +# make +# sudo make install +# +# - name: Install x264 +# shell: bash +# run: | +# set -x +# git clone --depth 1 https://code.videolan.org/videolan/x264.git -b "$X264_TAG" +# cd x264 +# ./configure \ +# --enable-static +# make +# sudo make install +# +# - name: Install x265 +# shell: bash +# run: | +# set -x +# hg clone http://hg.videolan.org/x265 -r "$X265_TAG" +# cd x265/build +# cmake ../source +# make +# sudo make install +# # This adjustment to the x265 linker flags is needed, at least on +# # arm, to successfully link against it statically. (-lgcc_s not +# # found (or needed), and -lpthread missing) +# sudo sed -e 's/-lgcc_s -lgcc -lgcc_s -lgcc/-lpthread -lgcc/' -i /usr/local/lib/pkgconfig/x265.pc +# +# - name: Install lame +# shell: bash +# run: | +# set -x +# curl -L -o lame-"$LAME_VERSION".tar.gz https://sourceforge.net/projects/lame/files/lame/"$LAME_VERSION"/lame-"$LAME_VERSION".tar.gz/download +# tar xzf lame-"$LAME_VERSION".tar.gz +# cd lame-"$LAME_VERSION" +# ./configure \ +# --enable-static \ +# --disable-shared +# make +# sudo make install +# +# - name: Install opus +# shell: bash +# run: | +# set -x +# curl -LO https://archive.mozilla.org/pub/opus/opus-"$OPUS_VERSION".tar.gz +# tar xzf opus-"$OPUS_VERSION".tar.gz +# cd opus-"$OPUS_VERSION" +# ./configure \ +# --enable-static \ +# --disable-shared +# make +# sudo make install +# # The pkgconfig linker flags for static opus don't work when ffmpeg +# # checks for opus in configure. Linking libm after libopus fixes it. +# sudo sed -e 's/-lopus/-lopus -lm/' -i /usr/local/lib/pkgconfig/opus.pc - name: Build ffmpeg and ffprobe shell: bash run: | set -x git clone --depth 1 https://git.ffmpeg.org/ffmpeg.git -b "$FFMPEG_TAG" - cd ffmpeg - ./configure \ - --pkg-config-flags="--static" \ - --extra-cflags="-static" \ - --extra-ldflags="-static" \ - --disable-ffplay \ - --enable-libvpx \ - --enable-libaom \ - --enable-libx264 \ - --enable-libx265 \ - --enable-libmp3lame \ - --enable-libopus \ - --enable-nvenc \ - --enable-vdpau \ - --enable-runtime-cpudetect \ - --enable-gpl \ - --enable-static - make - # Show that these are not dynamic executables. Fail if they are. - ldd ffmpeg && exit 1 - ldd ffprobe && exit 1 - # After commands that we expect to fail, we still need a successful - # command here to make this step a success. - true +# cd ffmpeg +# ./configure \ +# --pkg-config-flags="--static" \ +# --extra-cflags="-static" \ +# --extra-ldflags="-static" \ +# --disable-ffplay \ +# --enable-libvpx \ +# --enable-libaom \ +# --enable-libx264 \ +# --enable-libx265 \ +# --enable-libmp3lame \ +# --enable-libopus \ +# --enable-nvenc \ +# --enable-vdpau \ +# --enable-runtime-cpudetect \ +# --enable-gpl \ +# --enable-static +# make +# # Show that these are not dynamic executables. Fail if they are. +# ldd ffmpeg && exit 1 +# ldd ffprobe && exit 1 +# # After commands that we expect to fail, we still need a successful +# # command here to make this step a success. +# true - name: Prepare assets shell: bash @@ -190,8 +190,9 @@ jobs: set -x mkdir assets SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}${{ matrix.exe_ext}}" - cp ffmpeg/ffmpeg assets/ffmpeg"$SUFFIX" - cp ffmpeg/ffprobe assets/ffprobe-"$SUFFIX" +# cp ffmpeg/ffmpeg assets/ffmpeg"$SUFFIX" +# cp ffmpeg/ffprobe assets/ffprobe-"$SUFFIX" +# cp ffmpeg/README.md assets/ # FIXME: debugging # Show MD5 sums that can be verified by users later if they want to # check for authenticity. cd assets From 41a443a0569df3de1997c6e81b675ffe4344bfcb Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 15:20:01 -0700 Subject: [PATCH 020/181] Fix some poorly-commented lines in the build The previous commit resulted in a syntax error in the workflow. Hopefully this fixes it. How does one validate a workflow file without pushing to GitHub? --- .github/workflows/release.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 74535b3..ea234cc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -190,9 +190,9 @@ jobs: set -x mkdir assets SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}${{ matrix.exe_ext}}" -# cp ffmpeg/ffmpeg assets/ffmpeg"$SUFFIX" -# cp ffmpeg/ffprobe assets/ffprobe-"$SUFFIX" -# cp ffmpeg/README.md assets/ # FIXME: debugging + #cp ffmpeg/ffmpeg assets/ffmpeg"$SUFFIX" + #cp ffmpeg/ffprobe assets/ffprobe-"$SUFFIX" + cp ffmpeg/README.md assets/ # FIXME: debugging # Show MD5 sums that can be verified by users later if they want to # check for authenticity. cd assets From 931a770c9158d58b0ff43d3f91b985148a89bb74 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 15:24:03 -0700 Subject: [PATCH 021/181] Fix mime-type guessing in upload_all_assets Turns out I was misusing the mimetypes module. I should read docs more carefully. --- .github/workflows/custom-github-repo-api-action/action.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index d5eafde..fa508a4 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -83,7 +83,10 @@ runs: def upload_all_assets(release_id, folder): for asset in os.listdir(folder): asset_path = os.path.join(folder, asset) - mime_type = mimetypes.guess_type(asset_path) + # This returns a tuple of (type, encoding), where type can be None + mime_type = mimetypes.guess_type(asset_path)[0] + if not mime_type: + mime_type = 'application/octet-stream' upload_asset(release_id, asset_path, mime_type) def download_all_assets(release_id, output_path): From 041303ced4d8ea193bc114b9ae35651b68600dcd Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 15:26:38 -0700 Subject: [PATCH 022/181] Fix missing release_id in publish_release If you don't "need" a previous step directly, you can't access its outputs. --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ea234cc..400b33f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -211,7 +211,7 @@ jobs: publish_release: name: Publish release - needs: build + needs: [draft_release, build] runs-on: ubuntu-latest steps: - name: Checkout repo From e2a3a61f71ab859cd695bf1f1a584b817c530ca7 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 15:49:04 -0700 Subject: [PATCH 023/181] More debugging around release IDs I am no longer sure if the problem is in the steps that consume the ID, so now I am debugging the step that generates it. --- .github/workflows/custom-github-repo-api-action/action.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index fa508a4..268c266 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -70,7 +70,11 @@ runs: "draft": True, } body = call_api("/releases", data) - release_id = body["id"] + release_id = str(body["id"]) + # TODO: Cleanup debug + print("Release ID: {}".format(release_id) + print("Setting output release_id={}".format(release_id)) + # print("::set-output name=release_id::{}".format(release_id)) def upload_asset(release_id, asset_path, mime_type): From 445dc8a232ebda46fa79de8a7e8e2bee2935536a Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 15:50:06 -0700 Subject: [PATCH 024/181] Add missing paren in Python debugging code --- .github/workflows/custom-github-repo-api-action/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index 268c266..2738532 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -72,7 +72,7 @@ runs: body = call_api("/releases", data) release_id = str(body["id"]) # TODO: Cleanup debug - print("Release ID: {}".format(release_id) + print("Release ID: {}".format(release_id)) print("Setting output release_id={}".format(release_id)) # print("::set-output name=release_id::{}".format(release_id)) From 5956eb6d7908f27ed0544d7326985085acb164b0 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 16:02:29 -0700 Subject: [PATCH 025/181] Try declaring the release_id output in the custom action If that doesn't work, also try a new debugging technique to see if there are any outputs at all from the previous job. --- .../workflows/custom-github-repo-api-action/action.yaml | 8 ++++---- .github/workflows/release.yaml | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index 2738532..742e23a 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -15,6 +15,10 @@ inputs: description: A high-level method to invoke. See action source for details. required: true +outputs: + release_id: + description: The ID of a newly-created release, set by the draft_release method. + runs: using: composite steps: @@ -71,10 +75,6 @@ runs: } body = call_api("/releases", data) release_id = str(body["id"]) - # TODO: Cleanup debug - print("Release ID: {}".format(release_id)) - print("Setting output release_id={}".format(release_id)) - # print("::set-output name=release_id::{}".format(release_id)) def upload_asset(release_id, asset_path, mime_type): diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 400b33f..ef548c2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -202,6 +202,14 @@ jobs: shell: bash run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + # FIXME: debugging + - name: Debug previous job outputs + shell: bash + run: | + # What if we output all the outputs combined? What would we see? + # Thanks, https://stackoverflow.com/a/59201610 ! + echo "OUTPUTS: ${{join(needs.draft_release.outputs.*, '\n')}}" + - name: Attach assets to release uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: From 800ddad809c75c59076d559b94ef5cd2c590371a Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 16:06:54 -0700 Subject: [PATCH 026/181] Try distinct ids for jobs, steps, and outputs Nothing else has worked so far, so what if we try making the ids of jobs and steps distinct, as well as the job output ids and the step output ids? --- .github/workflows/release.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ef548c2..1aeae72 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -23,14 +23,14 @@ jobs: name: Draft release runs-on: ubuntu-latest outputs: - release_id: ${{ steps.draft_release.outputs.release_id }} + id: ${{ steps.draft_release_step.outputs.release_id }} steps: - name: Checkout repo shell: bash run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - name: Draft release - id: draft_release + id: draft_release_step uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: username: shaka-bot @@ -215,7 +215,7 @@ jobs: with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: upload_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/") + method: upload_all_assets("${{ needs.draft_release.outputs.id }}", "assets/") publish_release: name: Publish release @@ -231,7 +231,7 @@ jobs: with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: download_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/") + method: download_all_assets("${{ needs.draft_release.outputs.id }}", "assets/") - name: Checksum all release assets id: checksum @@ -249,4 +249,4 @@ jobs: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} # TODO: publish for real, draft=False - method: update_release("${{ needs.draft_release.outputs.release_id }}", { body="${{ steps.checksum.outputs.sums }}", draft=True }) + method: update_release("${{ needs.draft_release.outputs.id }}", { body="${{ steps.checksum.outputs.sums }}", draft=True }) From af071bf917b7622deaca55f006229b2bd4bcc321 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 16:09:36 -0700 Subject: [PATCH 027/181] Try printing step output from within job Why is this output ID missing? --- .github/workflows/release.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1aeae72..815bd5c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -37,6 +37,11 @@ jobs: token: ${{ secrets.SHAKA_BOT_TOKEN }} method: draft_release("${{ github.ref }}") + - name: Debug release ID from previous step + shell: bash + run: | + echo "step output: ${{ steps.draft_release_step.outputs.release_id }}" + build: needs: draft_release strategy: From badd4691976260cc891bf360ecd8418cdcb7e4ec Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 16:15:24 -0700 Subject: [PATCH 028/181] Additional debugging of step outputs I have seen the release ID printed from within the custom action, and the set-output command seems to work, insofar as it is swallowed by the runner and does not show up in the logs. So why is it not set when I need it in another job? Or in the same job? --- .github/workflows/release.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 815bd5c..e971183 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -37,10 +37,21 @@ jobs: token: ${{ secrets.SHAKA_BOT_TOKEN }} method: draft_release("${{ github.ref }}") - - name: Debug release ID from previous step + - name: Additional output variable for debugging + id: extra_debug_output shell: bash run: | - echo "step output: ${{ steps.draft_release_step.outputs.release_id }}" + echo "::set-output name=foo::bar" + echo "::set-output name=release_id_2::baz" + + - name: Debug outputs from previous steps + shell: bash + run: | + echo "output release_id: ${{ steps.draft_release_step.outputs.release_id }}" + echo "output foo: ${{ steps.extra_debug_output.outputs.foo }}" + echo "output release_id_2: ${{ steps.extra_debug_output.outputs.release_id_2 }}" + echo "all draft_release_step outputs: ${{ join(steps.draft_release_step.outputs.*, ' ') }}" + echo "all extra_debug_output outputs: ${{ join(steps.extra_debug_output.outputs.*, ' ') }}" build: needs: draft_release From 5e5731b93cc5d0fb4221020a6bbd4fb971fdb695 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 16:21:25 -0700 Subject: [PATCH 029/181] Fix export of output variables from custom action Just as a job needs to export outputs from a step, so an action needs to export outputs from one of its internal steps. This *should* fix it. Finally. Removing all the debug code and crossing my fingers. --- .../custom-github-repo-api-action/action.yaml | 6 +++++- .github/workflows/release.yaml | 19 ------------------- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index 742e23a..604763b 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -18,11 +18,13 @@ inputs: outputs: release_id: description: The ID of a newly-created release, set by the draft_release method. + value: ${{ steps.call-repo-api.outputs.release_id }} runs: using: composite steps: - name: Call Repo API + id: call-repo-api shell: python run: | import base64 @@ -40,7 +42,9 @@ runs: headers = {} if not method: method = "POST" if data else "GET" - print({'path': path, 'data': data, 'headers': headers, 'method': method}) + + # For debugging: + #print({'path': path, 'data': data, 'headers': headers, 'method': method}) auth_token = base64.b64encode(login.encode("utf-8")).decode("utf-8") headers["Authorization"] = "Basic {}".format(auth_token) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e971183..c1b9b80 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -37,21 +37,10 @@ jobs: token: ${{ secrets.SHAKA_BOT_TOKEN }} method: draft_release("${{ github.ref }}") - - name: Additional output variable for debugging - id: extra_debug_output - shell: bash - run: | - echo "::set-output name=foo::bar" - echo "::set-output name=release_id_2::baz" - - name: Debug outputs from previous steps shell: bash run: | echo "output release_id: ${{ steps.draft_release_step.outputs.release_id }}" - echo "output foo: ${{ steps.extra_debug_output.outputs.foo }}" - echo "output release_id_2: ${{ steps.extra_debug_output.outputs.release_id_2 }}" - echo "all draft_release_step outputs: ${{ join(steps.draft_release_step.outputs.*, ' ') }}" - echo "all extra_debug_output outputs: ${{ join(steps.extra_debug_output.outputs.*, ' ') }}" build: needs: draft_release @@ -218,14 +207,6 @@ jobs: shell: bash run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - # FIXME: debugging - - name: Debug previous job outputs - shell: bash - run: | - # What if we output all the outputs combined? What would we see? - # Thanks, https://stackoverflow.com/a/59201610 ! - echo "OUTPUTS: ${{join(needs.draft_release.outputs.*, '\n')}}" - - name: Attach assets to release uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: From 5e34b8c7c16dc7935450ad48cca65ca8de2fc053 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 16:24:54 -0700 Subject: [PATCH 030/181] Debug API requests again The upload request is still failing. The release_id mess is now fixed, and I can see it being sent to the upload step correctly, but likely there is another issue I haven't found yet. --- .../custom-github-repo-api-action/action.yaml | 3 ++- .github/workflows/release.yaml | 19 +++++++------------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index 604763b..8372148 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -44,7 +44,7 @@ runs: method = "POST" if data else "GET" # For debugging: - #print({'path': path, 'data': data, 'headers': headers, 'method': method}) + print({'path': path, 'data': data, 'headers': headers, 'method': method}) auth_token = base64.b64encode(login.encode("utf-8")).decode("utf-8") headers["Authorization"] = "Basic {}".format(auth_token) @@ -78,6 +78,7 @@ runs: "draft": True, } body = call_api("/releases", data) + print("Draft release body:", body) release_id = str(body["id"]) print("::set-output name=release_id::{}".format(release_id)) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c1b9b80..0421a18 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -23,25 +23,20 @@ jobs: name: Draft release runs-on: ubuntu-latest outputs: - id: ${{ steps.draft_release_step.outputs.release_id }} + release_id: ${{ steps.draft_release.outputs.release_id }} steps: - name: Checkout repo shell: bash run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - name: Draft release - id: draft_release_step + id: draft_release uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} method: draft_release("${{ github.ref }}") - - name: Debug outputs from previous steps - shell: bash - run: | - echo "output release_id: ${{ steps.draft_release_step.outputs.release_id }}" - build: needs: draft_release strategy: @@ -212,7 +207,7 @@ jobs: with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: upload_all_assets("${{ needs.draft_release.outputs.id }}", "assets/") + method: upload_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/") publish_release: name: Publish release @@ -228,10 +223,10 @@ jobs: with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: download_all_assets("${{ needs.draft_release.outputs.id }}", "assets/") + method: download_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/") - - name: Checksum all release assets - id: checksum + - name: Calculate MD5 sums of all release assets + id: md5sum shell: bash run: | cd assets @@ -246,4 +241,4 @@ jobs: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} # TODO: publish for real, draft=False - method: update_release("${{ needs.draft_release.outputs.id }}", { body="${{ steps.checksum.outputs.sums }}", draft=True }) + method: update_release("${{ needs.draft_release.outputs.release_id }}", { body="${{ steps.md5sum.outputs.sums }}", draft=True }) From ad47f1e08814571c166d46bcaafd264f6587c580 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 16:29:05 -0700 Subject: [PATCH 031/181] Correct upload URL I misread the upload_url field of the draft release when I wrote the upload method. --- .../workflows/custom-github-repo-api-action/action.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index 8372148..16bd543 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -33,9 +33,12 @@ runs: import os import urllib.request - def call_api(path, data=None, headers=None, method=None, get_json=True): + def call_api(path, data=None, headers=None, method=None, get_json=True, upload=False): repo = os.environ["GITHUB_REPOSITORY"] - url = "https://api.github.com/repos/{}{}".format(repo, path) + if upload: + url = "https://upload.github.com/repos/{}{}".format(repo, path) + else: + url = "https://api.github.com/repos/{}{}".format(repo, path) login = "${{ inputs.username }}:${{ inputs.token }}" if not headers: @@ -87,7 +90,7 @@ runs: path = "/releases/{}/assets?name={}".format(release_id, base_name) with open(asset_path, "rb") as f: data = f.read() - call_api(path, data, headers={"Content-Type": mime_type}) + call_api(path, data, headers={"Content-Type": mime_type}, upload=True) def upload_all_assets(release_id, folder): for asset in os.listdir(folder): From f31c66961bc574d192ae0e2fb181150d303f6e87 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 16:35:30 -0700 Subject: [PATCH 032/181] Try using HTTP PUT for uploads The docs are not very clear on this, IMO. --- .../custom-github-repo-api-action/action.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index 16bd543..9b12640 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -17,7 +17,7 @@ inputs: outputs: release_id: - description: The ID of a newly-created release, set by the draft_release method. + description: The ID of a newly-created release from the draft_release method value: ${{ steps.call-repo-api.outputs.release_id }} runs: @@ -33,7 +33,8 @@ runs: import os import urllib.request - def call_api(path, data=None, headers=None, method=None, get_json=True, upload=False): + def call_api(path, data=None, headers=None, method=None, get_json=True, + upload=False): repo = os.environ["GITHUB_REPOSITORY"] if upload: url = "https://upload.github.com/repos/{}{}".format(repo, path) @@ -47,7 +48,13 @@ runs: method = "POST" if data else "GET" # For debugging: - print({'path': path, 'data': data, 'headers': headers, 'method': method}) + print({ + "method": method, + "path": path, + "url": url, + "headers": headers, + "data": data, + }) auth_token = base64.b64encode(login.encode("utf-8")).decode("utf-8") headers["Authorization"] = "Basic {}".format(auth_token) @@ -90,7 +97,8 @@ runs: path = "/releases/{}/assets?name={}".format(release_id, base_name) with open(asset_path, "rb") as f: data = f.read() - call_api(path, data, headers={"Content-Type": mime_type}, upload=True) + headers = {"Content-Type": mime_type} + call_api(path, data, headers, upload=True, method="PUT") def upload_all_assets(release_id, folder): for asset in os.listdir(folder): @@ -98,7 +106,7 @@ runs: # This returns a tuple of (type, encoding), where type can be None mime_type = mimetypes.guess_type(asset_path)[0] if not mime_type: - mime_type = 'application/octet-stream' + mime_type = "application/octet-stream" upload_asset(release_id, asset_path, mime_type) def download_all_assets(release_id, output_path): From 74c8af459e0d144d7480ceb5fd2ccd3e9a4e0623 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 16:40:57 -0700 Subject: [PATCH 033/181] Correct upload URL again uploads (plural) dot github dot com. --- .github/workflows/custom-github-repo-api-action/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index 9b12640..22883a6 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -37,7 +37,7 @@ runs: upload=False): repo = os.environ["GITHUB_REPOSITORY"] if upload: - url = "https://upload.github.com/repos/{}{}".format(repo, path) + url = "https://uploads.github.com/repos/{}{}".format(repo, path) else: url = "https://api.github.com/repos/{}{}".format(repo, path) login = "${{ inputs.username }}:${{ inputs.token }}" @@ -98,7 +98,7 @@ runs: with open(asset_path, "rb") as f: data = f.read() headers = {"Content-Type": mime_type} - call_api(path, data, headers, upload=True, method="PUT") + call_api(path, data, headers, upload=True) def upload_all_assets(release_id, folder): for asset in os.listdir(folder): From 5fee121ed1d002ce743b0f015ee3a346db6b9620 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 16:42:51 -0700 Subject: [PATCH 034/181] Fix typo in download_all_assets --- .github/workflows/custom-github-repo-api-action/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index 22883a6..588069b 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -110,7 +110,7 @@ runs: upload_asset(release_id, asset_path, mime_type) def download_all_assets(release_id, output_path): - if not os.exists(output_path): + if not os.path.exists(output_path): os.mkdir(output_path) path = "/releases/{}/assets".format(release_id) From 8cb7f35069b7b30062c6e2c37a531c9df1f1c2fd Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 16:43:21 -0700 Subject: [PATCH 035/181] Disable some debugging code in custom action --- .../custom-github-repo-api-action/action.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index 588069b..dbf1592 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -48,13 +48,13 @@ runs: method = "POST" if data else "GET" # For debugging: - print({ - "method": method, - "path": path, - "url": url, - "headers": headers, - "data": data, - }) + #print({ + # "method": method, + # "path": path, + # "url": url, + # "headers": headers, + # "data": data, + #}) auth_token = base64.b64encode(login.encode("utf-8")).decode("utf-8") headers["Authorization"] = "Basic {}".format(auth_token) @@ -88,7 +88,7 @@ runs: "draft": True, } body = call_api("/releases", data) - print("Draft release body:", body) + #print("Draft release body:", body) release_id = str(body["id"]) print("::set-output name=release_id::{}".format(release_id)) From c1d8eeab16e332fb1392f57c81258c8ffac8d537 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 16:45:00 -0700 Subject: [PATCH 036/181] Revert "Disable some debugging code in custom action" This reverts commit 8cb7f35069b7b30062c6e2c37a531c9df1f1c2fd. --- .../custom-github-repo-api-action/action.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index dbf1592..588069b 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -48,13 +48,13 @@ runs: method = "POST" if data else "GET" # For debugging: - #print({ - # "method": method, - # "path": path, - # "url": url, - # "headers": headers, - # "data": data, - #}) + print({ + "method": method, + "path": path, + "url": url, + "headers": headers, + "data": data, + }) auth_token = base64.b64encode(login.encode("utf-8")).decode("utf-8") headers["Authorization"] = "Basic {}".format(auth_token) @@ -88,7 +88,7 @@ runs: "draft": True, } body = call_api("/releases", data) - #print("Draft release body:", body) + print("Draft release body:", body) release_id = str(body["id"]) print("::set-output name=release_id::{}".format(release_id)) From 030a02a5dcca76c1ca87c0bdaadd82bd7996498a Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 16:50:43 -0700 Subject: [PATCH 037/181] Sleep to wait for uploaded assets to become available May not be a real solution. I can fetch assets in curl, but time has passed by the time I can try it. So what if I wait 1 minute between upload and download? --- .github/workflows/release.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0421a18..2321de9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -218,6 +218,13 @@ jobs: shell: bash run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + - name: Sleep to wait for uploaded assets to become available + # TODO: May not be a real solution. I can fetch assets in curl, but + # time has passed by the time I can try it. So what if I wait 1 minute + # between upload and download? + shell: bash + run: sleep 60s + - name: Get all release assets uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: From 990607b8d1eeb7a393dca134b7dcb9a42a26d7b5 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 17:13:32 -0700 Subject: [PATCH 038/181] Revert "Sleep to wait for uploaded assets to become available" This reverts commit 030a02a5dcca76c1ca87c0bdaadd82bd7996498a. --- .github/workflows/release.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2321de9..0421a18 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -218,13 +218,6 @@ jobs: shell: bash run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - - name: Sleep to wait for uploaded assets to become available - # TODO: May not be a real solution. I can fetch assets in curl, but - # time has passed by the time I can try it. So what if I wait 1 minute - # between upload and download? - shell: bash - run: sleep 60s - - name: Get all release assets uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: From 0a49d46ea366c0690fb0a8634a3445278623b3fb Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 17:23:20 -0700 Subject: [PATCH 039/181] Fix tag names Having slashes in the release tag name is causing issues with the download URLs. --- .github/workflows/custom-github-repo-api-action/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index 588069b..e89fae8 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -80,11 +80,11 @@ runs: def draft_release(tag_name): # Turns "refs/tags/foo" into "foo" - name = tag_name.split("/").pop() + tag_name = tag_name.split("/").pop() data = { "tag_name": tag_name, - "name": name, + "name": tag_name, "draft": True, } body = call_api("/releases", data) From 1c5772c3116d648bfbfb1b31fb896542b3ee0606 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 17:25:19 -0700 Subject: [PATCH 040/181] Debug download URLs --- .github/workflows/custom-github-repo-api-action/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index e89fae8..a324161 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -118,6 +118,7 @@ runs: for asset in asset_list: name = asset["name"] url = asset["browser_download_url"] + print("Downloading: ", url) with urllib.request.urlopen(url) as download: with open(os.path.join(output_path, name), "wb") as local_file: local_file.write(download.read()) From f4e742aac7b3aa2e46adf8a41b10b5247372f0b3 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 17:25:32 -0700 Subject: [PATCH 041/181] Try publishing the release before downloading assets --- .github/workflows/release.yaml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0421a18..359616b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -218,6 +218,18 @@ jobs: shell: bash run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + # First, we have to take the release out of draft mode. Then, we can + # download the assets and use their MD5 sums to update the release notes. + # TODO: Verify that this is needed! It appears that we also need to fix + # the draft release tag to avoid refs/tags/ in it, because that ends up + # causing issues with the download URLs. + - name: Publish release + uses: ./repo-src/.github/workflows/custom-github-repo-api-action + with: + username: shaka-bot + token: ${{ secrets.SHAKA_BOT_TOKEN }} + method: update_release("${{ needs.draft_release.outputs.release_id }}", { draft=False }) + - name: Get all release assets uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: @@ -235,10 +247,9 @@ jobs: # The sums are now in an output variable that we can use in the next # step to set the release body. - - name: Publish release + - name: Update release notes uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - # TODO: publish for real, draft=False - method: update_release("${{ needs.draft_release.outputs.release_id }}", { body="${{ steps.md5sum.outputs.sums }}", draft=True }) + method: update_release("${{ needs.draft_release.outputs.release_id }}", { body="${{ steps.md5sum.outputs.sums }}" }) From bf47513cb61a1919f9849cf72e85b2cae84e8447 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 17:27:29 -0700 Subject: [PATCH 042/181] Fix syntax error in update_release calls --- .github/workflows/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 359616b..83a5cad 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -228,7 +228,7 @@ jobs: with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: update_release("${{ needs.draft_release.outputs.release_id }}", { draft=False }) + method: update_release("${{ needs.draft_release.outputs.release_id }}", { "draft": False }) - name: Get all release assets uses: ./repo-src/.github/workflows/custom-github-repo-api-action @@ -252,4 +252,4 @@ jobs: with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: update_release("${{ needs.draft_release.outputs.release_id }}", { body="${{ steps.md5sum.outputs.sums }}" }) + method: update_release("${{ needs.draft_release.outputs.release_id }}", { "body": "${{ steps.md5sum.outputs.sums }}" }) From 9e98af7647fed206a9b20dc12a10ccd8fc216e75 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 17:29:11 -0700 Subject: [PATCH 043/181] Quote all method strings Having a colon in the Python code in the with/method field seems to confuse the yaml parser. Try quoting the strings. --- .github/workflows/release.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 83a5cad..39addbf 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -35,7 +35,7 @@ jobs: with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: draft_release("${{ github.ref }}") + method: 'draft_release("${{ github.ref }}")' build: needs: draft_release @@ -207,7 +207,7 @@ jobs: with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: upload_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/") + method: 'upload_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/")' publish_release: name: Publish release @@ -228,14 +228,14 @@ jobs: with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: update_release("${{ needs.draft_release.outputs.release_id }}", { "draft": False }) + method: 'update_release("${{ needs.draft_release.outputs.release_id }}", { "draft": False })' - name: Get all release assets uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: download_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/") + method: 'download_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/")' - name: Calculate MD5 sums of all release assets id: md5sum @@ -252,4 +252,4 @@ jobs: with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: update_release("${{ needs.draft_release.outputs.release_id }}", { "body": "${{ steps.md5sum.outputs.sums }}" }) + method: 'update_release("${{ needs.draft_release.outputs.release_id }}", { "body": "${{ steps.md5sum.outputs.sums }}" })' From 61716ac61524e10dffedc86e060c78259df1be34 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 17:33:44 -0700 Subject: [PATCH 044/181] Fix use of kwargs in update_release I got confused about my own API. I was passing a dict instead of using kwargs. Doh! --- .github/workflows/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 39addbf..74e41c5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -228,7 +228,7 @@ jobs: with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: 'update_release("${{ needs.draft_release.outputs.release_id }}", { "draft": False })' + method: 'update_release("${{ needs.draft_release.outputs.release_id }}", draft=False)' - name: Get all release assets uses: ./repo-src/.github/workflows/custom-github-repo-api-action @@ -252,4 +252,4 @@ jobs: with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: 'update_release("${{ needs.draft_release.outputs.release_id }}", { "body": "${{ steps.md5sum.outputs.sums }}" })' + method: 'update_release("${{ needs.draft_release.outputs.release_id }}", body="${{ steps.md5sum.outputs.sums }}")' From 40cb132589b56eacf39ad4af120207ae2a1a6ba9 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 17:36:27 -0700 Subject: [PATCH 045/181] Try one more time to download before publication --- .github/workflows/release.yaml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 74e41c5..9935ecc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -218,18 +218,6 @@ jobs: shell: bash run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - # First, we have to take the release out of draft mode. Then, we can - # download the assets and use their MD5 sums to update the release notes. - # TODO: Verify that this is needed! It appears that we also need to fix - # the draft release tag to avoid refs/tags/ in it, because that ends up - # causing issues with the download URLs. - - name: Publish release - uses: ./repo-src/.github/workflows/custom-github-repo-api-action - with: - username: shaka-bot - token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: 'update_release("${{ needs.draft_release.outputs.release_id }}", draft=False)' - - name: Get all release assets uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: @@ -247,9 +235,9 @@ jobs: # The sums are now in an output variable that we can use in the next # step to set the release body. - - name: Update release notes + - name: Publish release uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: 'update_release("${{ needs.draft_release.outputs.release_id }}", body="${{ steps.md5sum.outputs.sums }}")' + method: 'update_release("${{ needs.draft_release.outputs.release_id }}", body="${{ steps.md5sum.outputs.sums }}", draft=False)' From b1fc055a00009ded0008f813f138cbd364dd7b4d Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 17:37:36 -0700 Subject: [PATCH 046/181] Revert "Try one more time to download before publication" This reverts commit 40cb132589b56eacf39ad4af120207ae2a1a6ba9. --- .github/workflows/release.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9935ecc..74e41c5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -218,6 +218,18 @@ jobs: shell: bash run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + # First, we have to take the release out of draft mode. Then, we can + # download the assets and use their MD5 sums to update the release notes. + # TODO: Verify that this is needed! It appears that we also need to fix + # the draft release tag to avoid refs/tags/ in it, because that ends up + # causing issues with the download URLs. + - name: Publish release + uses: ./repo-src/.github/workflows/custom-github-repo-api-action + with: + username: shaka-bot + token: ${{ secrets.SHAKA_BOT_TOKEN }} + method: 'update_release("${{ needs.draft_release.outputs.release_id }}", draft=False)' + - name: Get all release assets uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: @@ -235,9 +247,9 @@ jobs: # The sums are now in an output variable that we can use in the next # step to set the release body. - - name: Publish release + - name: Update release notes uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: 'update_release("${{ needs.draft_release.outputs.release_id }}", body="${{ steps.md5sum.outputs.sums }}", draft=False)' + method: 'update_release("${{ needs.draft_release.outputs.release_id }}", body="${{ steps.md5sum.outputs.sums }}")' From 3256579acef44532ef3ecca0e1b01f63ec529d56 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 17:49:28 -0700 Subject: [PATCH 047/181] Re-enable build steps Also disable debugging in API action --- .../custom-github-repo-api-action/action.yaml | 18 +- .github/workflows/release.yaml | 252 +++++++++--------- 2 files changed, 133 insertions(+), 137 deletions(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index a324161..c9bdc94 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -48,13 +48,13 @@ runs: method = "POST" if data else "GET" # For debugging: - print({ - "method": method, - "path": path, - "url": url, - "headers": headers, - "data": data, - }) + #print({ + # "method": method, + # "path": path, + # "url": url, + # "headers": headers, + # "data": data, + #}) auth_token = base64.b64encode(login.encode("utf-8")).decode("utf-8") headers["Authorization"] = "Basic {}".format(auth_token) @@ -88,7 +88,7 @@ runs: "draft": True, } body = call_api("/releases", data) - print("Draft release body:", body) + #print("Draft release body:", body) release_id = str(body["id"]) print("::set-output name=release_id::{}".format(release_id)) @@ -118,7 +118,7 @@ runs: for asset in asset_list: name = asset["name"] url = asset["browser_download_url"] - print("Downloading: ", url) + #print("Downloading: ", url) with urllib.request.urlopen(url) as download: with open(os.path.join(output_path, name), "wb") as local_file: local_file.write(download.read()) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 74e41c5..9945b50 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -56,133 +56,133 @@ jobs: runs-on: ${{ matrix.os }} steps: -# - name: Install packages -# if: runner.os == 'Linux' -# shell: bash -# run: | -# set -x -# sudo apt -y install \ -# nasm \ -# yasm \ -# libffmpeg-nvenc-dev \ -# libvdpau-dev -# -# - name: Install libvpx -# shell: bash -# run: | -# set -x -# git clone --depth 1 https://chromium.googlesource.com/webm/libvpx -b "$LIBVPX_TAG" -# cd libvpx -# ./configure \ -# --enable-vp8 \ -# --enable-vp9 \ -# --enable-runtime-cpu-detect \ -# --enable-static \ -# --disable-shared -# make -# sudo make install -# -# - name: Install aom -# shell: bash -# run: | -# set -x -# git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" -# mkdir aom_build -# cd aom_build -# cmake ../aom \ -# -DENABLE_DOCS=OFF \ -# -DENABLE_EXAMPLES=OFF \ -# -DENABLE_TESTS=OFF \ -# -DENABLE_TESTDATA=OFF \ -# -DENABLE_TOOLS=OFF \ -# -DCONFIG_RUNTIME_CPU_DETECT=1 \ -# -DCONFIG_SHARED=0 -# make -# sudo make install -# -# - name: Install x264 -# shell: bash -# run: | -# set -x -# git clone --depth 1 https://code.videolan.org/videolan/x264.git -b "$X264_TAG" -# cd x264 -# ./configure \ -# --enable-static -# make -# sudo make install -# -# - name: Install x265 -# shell: bash -# run: | -# set -x -# hg clone http://hg.videolan.org/x265 -r "$X265_TAG" -# cd x265/build -# cmake ../source -# make -# sudo make install -# # This adjustment to the x265 linker flags is needed, at least on -# # arm, to successfully link against it statically. (-lgcc_s not -# # found (or needed), and -lpthread missing) -# sudo sed -e 's/-lgcc_s -lgcc -lgcc_s -lgcc/-lpthread -lgcc/' -i /usr/local/lib/pkgconfig/x265.pc -# -# - name: Install lame -# shell: bash -# run: | -# set -x -# curl -L -o lame-"$LAME_VERSION".tar.gz https://sourceforge.net/projects/lame/files/lame/"$LAME_VERSION"/lame-"$LAME_VERSION".tar.gz/download -# tar xzf lame-"$LAME_VERSION".tar.gz -# cd lame-"$LAME_VERSION" -# ./configure \ -# --enable-static \ -# --disable-shared -# make -# sudo make install -# -# - name: Install opus -# shell: bash -# run: | -# set -x -# curl -LO https://archive.mozilla.org/pub/opus/opus-"$OPUS_VERSION".tar.gz -# tar xzf opus-"$OPUS_VERSION".tar.gz -# cd opus-"$OPUS_VERSION" -# ./configure \ -# --enable-static \ -# --disable-shared -# make -# sudo make install -# # The pkgconfig linker flags for static opus don't work when ffmpeg -# # checks for opus in configure. Linking libm after libopus fixes it. -# sudo sed -e 's/-lopus/-lopus -lm/' -i /usr/local/lib/pkgconfig/opus.pc + - name: Install packages + if: runner.os == 'Linux' + shell: bash + run: | + set -x + sudo apt -y install \ + nasm \ + yasm \ + libffmpeg-nvenc-dev \ + libvdpau-dev + + - name: Install libvpx + shell: bash + run: | + set -x + git clone --depth 1 https://chromium.googlesource.com/webm/libvpx -b "$LIBVPX_TAG" + cd libvpx + ./configure \ + --enable-vp8 \ + --enable-vp9 \ + --enable-runtime-cpu-detect \ + --enable-static \ + --disable-shared + make + sudo make install + + - name: Install aom + shell: bash + run: | + set -x + git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" + mkdir aom_build + cd aom_build + cmake ../aom \ + -DENABLE_DOCS=OFF \ + -DENABLE_EXAMPLES=OFF \ + -DENABLE_TESTS=OFF \ + -DENABLE_TESTDATA=OFF \ + -DENABLE_TOOLS=OFF \ + -DCONFIG_RUNTIME_CPU_DETECT=1 \ + -DCONFIG_SHARED=0 + make + sudo make install + + - name: Install x264 + shell: bash + run: | + set -x + git clone --depth 1 https://code.videolan.org/videolan/x264.git -b "$X264_TAG" + cd x264 + ./configure \ + --enable-static + make + sudo make install + + - name: Install x265 + shell: bash + run: | + set -x + hg clone http://hg.videolan.org/x265 -r "$X265_TAG" + cd x265/build + cmake ../source + make + sudo make install + # This adjustment to the x265 linker flags is needed, at least on + # arm, to successfully link against it statically. (-lgcc_s not + # found (or needed), and -lpthread missing) + sudo sed -e 's/-lgcc_s -lgcc -lgcc_s -lgcc/-lpthread -lgcc/' -i /usr/local/lib/pkgconfig/x265.pc + + - name: Install lame + shell: bash + run: | + set -x + curl -L -o lame-"$LAME_VERSION".tar.gz https://sourceforge.net/projects/lame/files/lame/"$LAME_VERSION"/lame-"$LAME_VERSION".tar.gz/download + tar xzf lame-"$LAME_VERSION".tar.gz + cd lame-"$LAME_VERSION" + ./configure \ + --enable-static \ + --disable-shared + make + sudo make install + + - name: Install opus + shell: bash + run: | + set -x + curl -LO https://archive.mozilla.org/pub/opus/opus-"$OPUS_VERSION".tar.gz + tar xzf opus-"$OPUS_VERSION".tar.gz + cd opus-"$OPUS_VERSION" + ./configure \ + --enable-static \ + --disable-shared + make + sudo make install + # The pkgconfig linker flags for static opus don't work when ffmpeg + # checks for opus in configure. Linking libm after libopus fixes it. + sudo sed -e 's/-lopus/-lopus -lm/' -i /usr/local/lib/pkgconfig/opus.pc - name: Build ffmpeg and ffprobe shell: bash run: | set -x git clone --depth 1 https://git.ffmpeg.org/ffmpeg.git -b "$FFMPEG_TAG" -# cd ffmpeg -# ./configure \ -# --pkg-config-flags="--static" \ -# --extra-cflags="-static" \ -# --extra-ldflags="-static" \ -# --disable-ffplay \ -# --enable-libvpx \ -# --enable-libaom \ -# --enable-libx264 \ -# --enable-libx265 \ -# --enable-libmp3lame \ -# --enable-libopus \ -# --enable-nvenc \ -# --enable-vdpau \ -# --enable-runtime-cpudetect \ -# --enable-gpl \ -# --enable-static -# make -# # Show that these are not dynamic executables. Fail if they are. -# ldd ffmpeg && exit 1 -# ldd ffprobe && exit 1 -# # After commands that we expect to fail, we still need a successful -# # command here to make this step a success. -# true + cd ffmpeg + ./configure \ + --pkg-config-flags="--static" \ + --extra-cflags="-static" \ + --extra-ldflags="-static" \ + --disable-ffplay \ + --enable-libvpx \ + --enable-libaom \ + --enable-libx264 \ + --enable-libx265 \ + --enable-libmp3lame \ + --enable-libopus \ + --enable-nvenc \ + --enable-vdpau \ + --enable-runtime-cpudetect \ + --enable-gpl \ + --enable-static + make + # Show that these are not dynamic executables. Fail if they are. + ldd ffmpeg && exit 1 + ldd ffprobe && exit 1 + # After commands that we expect to fail, we still need a successful + # command here to make this step a success. + true - name: Prepare assets shell: bash @@ -190,9 +190,8 @@ jobs: set -x mkdir assets SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}${{ matrix.exe_ext}}" - #cp ffmpeg/ffmpeg assets/ffmpeg"$SUFFIX" - #cp ffmpeg/ffprobe assets/ffprobe-"$SUFFIX" - cp ffmpeg/README.md assets/ # FIXME: debugging + cp ffmpeg/ffmpeg assets/ffmpeg"$SUFFIX" + cp ffmpeg/ffprobe assets/ffprobe-"$SUFFIX" # Show MD5 sums that can be verified by users later if they want to # check for authenticity. cd assets @@ -220,9 +219,6 @@ jobs: # First, we have to take the release out of draft mode. Then, we can # download the assets and use their MD5 sums to update the release notes. - # TODO: Verify that this is needed! It appears that we also need to fix - # the draft release tag to avoid refs/tags/ in it, because that ends up - # causing issues with the download URLs. - name: Publish release uses: ./repo-src/.github/workflows/custom-github-repo-api-action with: @@ -252,4 +248,4 @@ jobs: with: username: shaka-bot token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: 'update_release("${{ needs.draft_release.outputs.release_id }}", body="${{ steps.md5sum.outputs.sums }}")' + method: 'update_release("${{ needs.draft_release.outputs.release_id }}", body="MD5 sums:\n${{ steps.md5sum.outputs.sums }}")' From a7b0ec661cb995a1315b940165f4c4477ab319ed Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 17:50:11 -0700 Subject: [PATCH 048/181] Enable mac --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9945b50..f77895a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -45,7 +45,7 @@ jobs: # TODO: Mac # TODO: arm64 #os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] - os: ["ubuntu-latest"] + os: ["ubuntu-latest", "macos-latest"] include: - os: ubuntu-latest os_name: linux From 073106ca4b6bcd97c3ca5202797db6ecb8a43093 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 19:11:12 -0700 Subject: [PATCH 049/181] Add macOS packages --- .github/workflows/release.yaml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f77895a..4a423eb 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -56,7 +56,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Install packages + - name: Install Linux packages if: runner.os == 'Linux' shell: bash run: | @@ -67,6 +67,22 @@ jobs: libffmpeg-nvenc-dev \ libvdpau-dev + - name: Install macOS packages + if: runner.os == 'macOS' + shell: bash + run: | + set -x + brew install \ + nasm \ + yasm + + - name: Install Windows packages + if: runner.os == 'Windows' + shell: bash + run: | + set -x + true # FIXME: install prereqs & tools on Windows + - name: Install libvpx shell: bash run: | From c63bc1cb849d3bca996811bf2b9c182cdf4dd852 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 19:12:24 -0700 Subject: [PATCH 050/181] Add steps to debug failures over SSH --- .github/workflows/release.yaml | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4a423eb..1e0c565 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -76,6 +76,12 @@ jobs: nasm \ yasm + - name: Debug + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + if: failure() + - name: Install Windows packages if: runner.os == 'Windows' shell: bash @@ -98,6 +104,12 @@ jobs: make sudo make install + - name: Debug + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + if: failure() + - name: Install aom shell: bash run: | @@ -116,6 +128,12 @@ jobs: make sudo make install + - name: Debug + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + if: failure() + - name: Install x264 shell: bash run: | @@ -127,6 +145,12 @@ jobs: make sudo make install + - name: Debug + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + if: failure() + - name: Install x265 shell: bash run: | @@ -141,6 +165,12 @@ jobs: # found (or needed), and -lpthread missing) sudo sed -e 's/-lgcc_s -lgcc -lgcc_s -lgcc/-lpthread -lgcc/' -i /usr/local/lib/pkgconfig/x265.pc + - name: Debug + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + if: failure() + - name: Install lame shell: bash run: | @@ -154,6 +184,12 @@ jobs: make sudo make install + - name: Debug + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + if: failure() + - name: Install opus shell: bash run: | @@ -170,6 +206,12 @@ jobs: # checks for opus in configure. Linking libm after libopus fixes it. sudo sed -e 's/-lopus/-lopus -lm/' -i /usr/local/lib/pkgconfig/opus.pc + - name: Debug + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + if: failure() + - name: Build ffmpeg and ffprobe shell: bash run: | @@ -200,6 +242,12 @@ jobs: # command here to make this step a success. true + - name: Debug + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + if: failure() + - name: Prepare assets shell: bash run: | From 774a6e5e784a72630d25aecba570462b1c511f76 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 19:14:28 -0700 Subject: [PATCH 051/181] Add missing details for macos in build matrix --- .github/workflows/release.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1e0c565..e984c6d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -51,6 +51,10 @@ jobs: os_name: linux target_arch: x64 exe_ext: "" + - os: macos-latest + os_name: osx + target_arch: x64 + exe_ext: "" name: Build ${{ matrix.os_name }} ${{ matrix.target_arch }} runs-on: ${{ matrix.os }} From 4c91cc83e865f1306b826eb0bce585a0c4a7afac Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 19:32:21 -0700 Subject: [PATCH 052/181] Install mercurial on mac --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e984c6d..d37ea51 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -77,6 +77,7 @@ jobs: run: | set -x brew install \ + mercurial \ nasm \ yasm From c51a95b79760caadb70e1f20242ac8480fbae800 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 20:14:50 -0700 Subject: [PATCH 053/181] Fix sed commands to work on macos Apparently the -i option for sed on mac requires an extension for a backup file. --- .github/workflows/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d37ea51..649fe35 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -168,7 +168,7 @@ jobs: # This adjustment to the x265 linker flags is needed, at least on # arm, to successfully link against it statically. (-lgcc_s not # found (or needed), and -lpthread missing) - sudo sed -e 's/-lgcc_s -lgcc -lgcc_s -lgcc/-lpthread -lgcc/' -i /usr/local/lib/pkgconfig/x265.pc + sudo sed -e 's/-lgcc_s -lgcc -lgcc_s -lgcc/-lpthread -lgcc/' -i.bk /usr/local/lib/pkgconfig/x265.pc - name: Debug uses: mxschmitt/action-tmate@v3 @@ -209,7 +209,7 @@ jobs: sudo make install # The pkgconfig linker flags for static opus don't work when ffmpeg # checks for opus in configure. Linking libm after libopus fixes it. - sudo sed -e 's/-lopus/-lopus -lm/' -i /usr/local/lib/pkgconfig/opus.pc + sudo sed -e 's/-lopus/-lopus -lm/' -i.bk /usr/local/lib/pkgconfig/opus.pc - name: Debug uses: mxschmitt/action-tmate@v3 From 054c0e9653a28ebe4acafdd13d12369ed8f6816a Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 21:33:11 -0700 Subject: [PATCH 054/181] Use different ldflags for linux and macos --- .github/workflows/release.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 649fe35..82e5b56 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -223,10 +223,16 @@ jobs: set -x git clone --depth 1 https://git.ffmpeg.org/ffmpeg.git -b "$FFMPEG_TAG" cd ffmpeg + if [[ "${{ runner.os }}" == "Linux" ]]; then + export CFLAGS="-static" + export LDFLAGS="-static" + elif [[ "${{ runner.os }}" == "macOS" ]]; then + export CFLAGS="-static" + # You can't do a _truly_ static build on macOS except the kernel. + # See https://stackoverflow.com/a/3801032 + fi ./configure \ --pkg-config-flags="--static" \ - --extra-cflags="-static" \ - --extra-ldflags="-static" \ --disable-ffplay \ --enable-libvpx \ --enable-libaom \ From 484e2bc83046e7f1d940571d5c54306eb48f0dcf Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 21:52:40 -0700 Subject: [PATCH 055/181] Remove extra dash from ffprobe name --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 82e5b56..06d305f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -266,7 +266,7 @@ jobs: mkdir assets SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}${{ matrix.exe_ext}}" cp ffmpeg/ffmpeg assets/ffmpeg"$SUFFIX" - cp ffmpeg/ffprobe assets/ffprobe-"$SUFFIX" + cp ffmpeg/ffprobe assets/ffprobe"$SUFFIX" # Show MD5 sums that can be verified by users later if they want to # check for authenticity. cd assets From 8a5bc317b498c273abcaf0a47f61974605a5f2aa Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 22:14:32 -0700 Subject: [PATCH 056/181] Disable SSH debugging --- .github/workflows/release.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 06d305f..a678cde 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -85,7 +85,7 @@ jobs: uses: mxschmitt/action-tmate@v3 with: limit-access-to-actor: true - if: failure() + if: 0 && failure() - name: Install Windows packages if: runner.os == 'Windows' @@ -113,7 +113,7 @@ jobs: uses: mxschmitt/action-tmate@v3 with: limit-access-to-actor: true - if: failure() + if: 0 && failure() - name: Install aom shell: bash @@ -137,7 +137,7 @@ jobs: uses: mxschmitt/action-tmate@v3 with: limit-access-to-actor: true - if: failure() + if: 0 && failure() - name: Install x264 shell: bash @@ -154,7 +154,7 @@ jobs: uses: mxschmitt/action-tmate@v3 with: limit-access-to-actor: true - if: failure() + if: 0 && failure() - name: Install x265 shell: bash @@ -174,7 +174,7 @@ jobs: uses: mxschmitt/action-tmate@v3 with: limit-access-to-actor: true - if: failure() + if: 0 && failure() - name: Install lame shell: bash @@ -193,7 +193,7 @@ jobs: uses: mxschmitt/action-tmate@v3 with: limit-access-to-actor: true - if: failure() + if: 0 && failure() - name: Install opus shell: bash @@ -215,7 +215,7 @@ jobs: uses: mxschmitt/action-tmate@v3 with: limit-access-to-actor: true - if: failure() + if: 0 && failure() - name: Build ffmpeg and ffprobe shell: bash @@ -257,7 +257,7 @@ jobs: uses: mxschmitt/action-tmate@v3 with: limit-access-to-actor: true - if: failure() + if: 0 && failure() - name: Prepare assets shell: bash From 97750024fbc12591c915b6d2e8eb0e8a36a93b52 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 22:14:49 -0700 Subject: [PATCH 057/181] Only configure ffmpeg with nvenc and vdpau on Linux --- .github/workflows/release.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a678cde..61f7a22 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -226,10 +226,12 @@ jobs: if [[ "${{ runner.os }}" == "Linux" ]]; then export CFLAGS="-static" export LDFLAGS="-static" + PLATFORM_CONFIGURE_FLAGS="--enable-nvenc --enable-vdpau" elif [[ "${{ runner.os }}" == "macOS" ]]; then export CFLAGS="-static" # You can't do a _truly_ static build on macOS except the kernel. - # See https://stackoverflow.com/a/3801032 + # So don't set LDFLAGS. See https://stackoverflow.com/a/3801032 + PLATFORM_CONFIGURE_FLAGS="" fi ./configure \ --pkg-config-flags="--static" \ @@ -240,11 +242,10 @@ jobs: --enable-libx265 \ --enable-libmp3lame \ --enable-libopus \ - --enable-nvenc \ - --enable-vdpau \ --enable-runtime-cpudetect \ --enable-gpl \ - --enable-static + --enable-static \ + $PLATFORM_CONFIGURE_FLAGS make # Show that these are not dynamic executables. Fail if they are. ldd ffmpeg && exit 1 From 0c748f29947af2a6f0b9f7e767491409e7152047 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 22:24:36 -0700 Subject: [PATCH 058/181] Improve the README --- README.md | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b190d91..cad3850 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,41 @@ # static-ffmpeg-binaries -Static binaries of FFmpeg, for multiple OS & CPU combinations, built from source in a GitHub Actions workflow. -The GitHub Actions workflows in this repo are what is covered by the Apache license. +Static binaries of FFmpeg, for multiple OS & CPU combinations, built from +source in a GitHub Actions workflow. -The resulting FFmpeg binaries are built using GPL libraries, and are therefore published under the GPL license. -Please see the [releases page](https://github.com/joeyparrish/static-ffmpeg-binaries/releases/) for binaries. +To download binaries, visit the [releases page][releases]. + + +# License + +The GitHub Actions workflows in this repo are covered by the Apache license. +Please see the [workflow source][source], and see [the Apache license][apache] +for license details. + +The resulting FFmpeg binaries are built using GPL libraries, and are therefore +published under the GPL license. +Please see the [releases page][releases] for binaries, and see [FFmpeg's GPL +license][gpl] for license details. + + +# How are they built? + +FFmpeg and its key dependencies are all built from source and linked statically. +Each run of the GitHub Actions workflow logs the MD5 sums of the binaries, and +it places the MD5 sums into the release notes. You can see how they were built, +and you can verify that they haven't been tampered with. The sums in the +workflow logs, release notes, and the binaries should all match. +You can read the details in the [workflow source][source]. + + +# Triggering a build + +Update the version numbers as needed in the [workflow][source], then create a +tag on the new commit. Full builds will be triggered, and binaries will be +attached to a release on the new tag. + + +[releases]: https://github.com/joeyparrish/static-ffmpeg-binaries/releases +[source]: https://github.com/joeyparrish/static-ffmpeg-binaries/blob/main/.github/workflows/release.yaml +[apache]: https://github.com/joeyparrish/static-ffmpeg-binaries/blob/main/LICENSE +[gpl]: https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.GPLv3 From a4bf4895253dbc06f578b6b04a0d031e8e529c61 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 6 Aug 2021 22:26:07 -0700 Subject: [PATCH 059/181] Add set -e to bash scripts to avoid masking errors --- .github/workflows/release.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 61f7a22..831a943 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -64,6 +64,7 @@ jobs: if: runner.os == 'Linux' shell: bash run: | + set -e set -x sudo apt -y install \ nasm \ @@ -75,6 +76,7 @@ jobs: if: runner.os == 'macOS' shell: bash run: | + set -e set -x brew install \ mercurial \ @@ -91,12 +93,14 @@ jobs: if: runner.os == 'Windows' shell: bash run: | + set -e set -x true # FIXME: install prereqs & tools on Windows - name: Install libvpx shell: bash run: | + set -e set -x git clone --depth 1 https://chromium.googlesource.com/webm/libvpx -b "$LIBVPX_TAG" cd libvpx @@ -118,6 +122,7 @@ jobs: - name: Install aom shell: bash run: | + set -e set -x git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" mkdir aom_build @@ -142,6 +147,7 @@ jobs: - name: Install x264 shell: bash run: | + set -e set -x git clone --depth 1 https://code.videolan.org/videolan/x264.git -b "$X264_TAG" cd x264 @@ -159,6 +165,7 @@ jobs: - name: Install x265 shell: bash run: | + set -e set -x hg clone http://hg.videolan.org/x265 -r "$X265_TAG" cd x265/build @@ -179,6 +186,7 @@ jobs: - name: Install lame shell: bash run: | + set -e set -x curl -L -o lame-"$LAME_VERSION".tar.gz https://sourceforge.net/projects/lame/files/lame/"$LAME_VERSION"/lame-"$LAME_VERSION".tar.gz/download tar xzf lame-"$LAME_VERSION".tar.gz @@ -198,6 +206,7 @@ jobs: - name: Install opus shell: bash run: | + set -e set -x curl -LO https://archive.mozilla.org/pub/opus/opus-"$OPUS_VERSION".tar.gz tar xzf opus-"$OPUS_VERSION".tar.gz @@ -220,6 +229,7 @@ jobs: - name: Build ffmpeg and ffprobe shell: bash run: | + set -e set -x git clone --depth 1 https://git.ffmpeg.org/ffmpeg.git -b "$FFMPEG_TAG" cd ffmpeg @@ -263,6 +273,7 @@ jobs: - name: Prepare assets shell: bash run: | + set -e set -x mkdir assets SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}${{ matrix.exe_ext}}" @@ -313,6 +324,8 @@ jobs: id: md5sum shell: bash run: | + set -e + set -x cd assets SUMS=$(md5sum *) echo "::set-output name=sums::$SUMS" From 9fea5efd9a259008f32e771655d385885f993c6c Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 08:23:59 -0700 Subject: [PATCH 060/181] Enable GPLv3 Now ffmpeg configure agrees with the docs on licensing --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 831a943..f0163f6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -246,6 +246,7 @@ jobs: ./configure \ --pkg-config-flags="--static" \ --disable-ffplay \ + --enable-version3 \ --enable-libvpx \ --enable-libaom \ --enable-libx264 \ From bf4011eae6cf5a26e425ae3f1483b6d81823abfb Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 08:24:30 -0700 Subject: [PATCH 061/181] Enable SSH debugging after ffmpeg build The build is failing on mac with an error about using x86 32-bit assembly. Not sure what to do yet. --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f0163f6..1fbbea2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -269,7 +269,7 @@ jobs: uses: mxschmitt/action-tmate@v3 with: limit-access-to-actor: true - if: 0 && failure() + if: failure() - name: Prepare assets shell: bash From 71145dfd96e31be079543563311c1f9c0ad6fd4d Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 08:25:35 -0700 Subject: [PATCH 062/181] Enable hardware encoding on mac --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1fbbea2..6dcc7e9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -241,7 +241,7 @@ jobs: export CFLAGS="-static" # You can't do a _truly_ static build on macOS except the kernel. # So don't set LDFLAGS. See https://stackoverflow.com/a/3801032 - PLATFORM_CONFIGURE_FLAGS="" + PLATFORM_CONFIGURE_FLAGS="--enable-videotoolbox" fi ./configure \ --pkg-config-flags="--static" \ From dae89656c41fbd9f07095dd7d64bbe7e0ffb455b Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 08:28:00 -0700 Subject: [PATCH 063/181] Rearrange ffmpeg configure flags --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6dcc7e9..026cd0b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -246,7 +246,6 @@ jobs: ./configure \ --pkg-config-flags="--static" \ --disable-ffplay \ - --enable-version3 \ --enable-libvpx \ --enable-libaom \ --enable-libx264 \ @@ -255,6 +254,7 @@ jobs: --enable-libopus \ --enable-runtime-cpudetect \ --enable-gpl \ + --enable-version3 \ --enable-static \ $PLATFORM_CONFIGURE_FLAGS make From 728bf52682aefda1731e8550a5cf7602a8662c68 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 09:14:15 -0700 Subject: [PATCH 064/181] Try using yasm on macos instead of nasm Based on https://wiki.libav.org/Platform/MacOSX --- .github/workflows/release.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 026cd0b..9931899 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -80,7 +80,6 @@ jobs: set -x brew install \ mercurial \ - nasm \ yasm - name: Debug From 7b4b5390791a34e0f04c9376275e2d23d6e8c4d4 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 09:31:09 -0700 Subject: [PATCH 065/181] Try uninstalling nasm before building ffmpeg We still need nasm earlier for x264 --- .github/workflows/release.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9931899..b307b96 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -80,6 +80,7 @@ jobs: set -x brew install \ mercurial \ + nasm \ yasm - name: Debug @@ -107,6 +108,7 @@ jobs: --enable-vp8 \ --enable-vp9 \ --enable-runtime-cpu-detect \ + --disable-unit-tests \ --enable-static \ --disable-shared make @@ -126,7 +128,9 @@ jobs: git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" mkdir aom_build cd aom_build + # NOTE: AOM_TEST_TEST_CMAKE_ is meant to disable the building of tests cmake ../aom \ + -DAOM_TEST_TEST_CMAKE_=1 \ -DENABLE_DOCS=OFF \ -DENABLE_EXAMPLES=OFF \ -DENABLE_TESTS=OFF \ @@ -241,6 +245,9 @@ jobs: # You can't do a _truly_ static build on macOS except the kernel. # So don't set LDFLAGS. See https://stackoverflow.com/a/3801032 PLATFORM_CONFIGURE_FLAGS="--enable-videotoolbox" + # Building with nasm seems to fail, so uninstall it and make ffmpeg + # use yasm instead. We needed nasm before to build x264, though. + brew uninstall nasm fi ./configure \ --pkg-config-flags="--static" \ From e4146114ba80feb757ab91086ac65c37e51a391a Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 09:35:07 -0700 Subject: [PATCH 066/181] Revert some changes that should not have been in the previous commit I was working on disabling unit tests to speed up the build, but accidentally added those changes to the nasm/yasm/macos changes. --- .github/workflows/release.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b307b96..4d4851b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -108,7 +108,6 @@ jobs: --enable-vp8 \ --enable-vp9 \ --enable-runtime-cpu-detect \ - --disable-unit-tests \ --enable-static \ --disable-shared make @@ -128,9 +127,7 @@ jobs: git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" mkdir aom_build cd aom_build - # NOTE: AOM_TEST_TEST_CMAKE_ is meant to disable the building of tests cmake ../aom \ - -DAOM_TEST_TEST_CMAKE_=1 \ -DENABLE_DOCS=OFF \ -DENABLE_EXAMPLES=OFF \ -DENABLE_TESTS=OFF \ From 3c27f59fecbeb251be512fcc4515aef2f19d695f Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 09:40:32 -0700 Subject: [PATCH 067/181] Delete unnecessary SSH-debug steps It turns out that one step at the end which runs on failure is enough to catch any failure at any earlier step. --- .github/workflows/release.yaml | 42 ---------------------------------- 1 file changed, 42 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4d4851b..fa49e11 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -83,12 +83,6 @@ jobs: nasm \ yasm - - name: Debug - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true - if: 0 && failure() - - name: Install Windows packages if: runner.os == 'Windows' shell: bash @@ -113,12 +107,6 @@ jobs: make sudo make install - - name: Debug - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true - if: 0 && failure() - - name: Install aom shell: bash run: | @@ -138,12 +126,6 @@ jobs: make sudo make install - - name: Debug - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true - if: 0 && failure() - - name: Install x264 shell: bash run: | @@ -156,12 +138,6 @@ jobs: make sudo make install - - name: Debug - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true - if: 0 && failure() - - name: Install x265 shell: bash run: | @@ -177,12 +153,6 @@ jobs: # found (or needed), and -lpthread missing) sudo sed -e 's/-lgcc_s -lgcc -lgcc_s -lgcc/-lpthread -lgcc/' -i.bk /usr/local/lib/pkgconfig/x265.pc - - name: Debug - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true - if: 0 && failure() - - name: Install lame shell: bash run: | @@ -197,12 +167,6 @@ jobs: make sudo make install - - name: Debug - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true - if: 0 && failure() - - name: Install opus shell: bash run: | @@ -220,12 +184,6 @@ jobs: # checks for opus in configure. Linking libm after libopus fixes it. sudo sed -e 's/-lopus/-lopus -lm/' -i.bk /usr/local/lib/pkgconfig/opus.pc - - name: Debug - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true - if: 0 && failure() - - name: Build ffmpeg and ffprobe shell: bash run: | From e6982cad6c8e244897b781051f7c37cfa6399e25 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 09:44:54 -0700 Subject: [PATCH 068/181] Add comment --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index fa49e11..58741aa 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -202,6 +202,7 @@ jobs: PLATFORM_CONFIGURE_FLAGS="--enable-videotoolbox" # Building with nasm seems to fail, so uninstall it and make ffmpeg # use yasm instead. We needed nasm before to build x264, though. + # See also https://wiki.libav.org/Platform/MacOSX brew uninstall nasm fi ./configure \ From 4362b719a19bfb3391cc343cb6a5a55951a78902 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 10:06:20 -0700 Subject: [PATCH 069/181] Disable x86 assembly on macos It fails to build with an error about how macho64 format can't contain 32-bit assembly. I'm not sure how else to resolve this, and from my searches, it appears that others are not having this problem with ffmpeg. --- .github/workflows/release.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 58741aa..e3bdafb 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -194,16 +194,24 @@ jobs: if [[ "${{ runner.os }}" == "Linux" ]]; then export CFLAGS="-static" export LDFLAGS="-static" + + # Enable platform-specific hardware acceleration. PLATFORM_CONFIGURE_FLAGS="--enable-nvenc --enable-vdpau" elif [[ "${{ runner.os }}" == "macOS" ]]; then export CFLAGS="-static" # You can't do a _truly_ static build on macOS except the kernel. # So don't set LDFLAGS. See https://stackoverflow.com/a/3801032 + + # Enable platform-specific hardware acceleration. PLATFORM_CONFIGURE_FLAGS="--enable-videotoolbox" - # Building with nasm seems to fail, so uninstall it and make ffmpeg - # use yasm instead. We needed nasm before to build x264, though. - # See also https://wiki.libav.org/Platform/MacOSX - brew uninstall nasm + + # Disable x86 ASM on macOS. It fails to build with an error about + # how macho64 format can't contain 32-bit assembly. I'm not sure + # how else to resolve this, and from my searches, it appears that + # others are not having this problem with ffmpeg. + # TODO: Try building from master branch to see if this has been + # resolved more recently than n4.4. + PLATFORM_CONFIGURE_FLAGS="$PLATFORM_CONFIGURE_FLAGS --disable-x86asm fi ./configure \ --pkg-config-flags="--static" \ From 79474eb4e47339e500a8381fa39c1f8d08fd0688 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 10:33:36 -0700 Subject: [PATCH 070/181] Add missing quote in platform config flags --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e3bdafb..3eef632 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -211,7 +211,7 @@ jobs: # others are not having this problem with ffmpeg. # TODO: Try building from master branch to see if this has been # resolved more recently than n4.4. - PLATFORM_CONFIGURE_FLAGS="$PLATFORM_CONFIGURE_FLAGS --disable-x86asm + PLATFORM_CONFIGURE_FLAGS="$PLATFORM_CONFIGURE_FLAGS --disable-x86asm" fi ./configure \ --pkg-config-flags="--static" \ From 606539f154e14b6bd242ba9d82146c344c668c38 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 10:56:58 -0700 Subject: [PATCH 071/181] Speed up libvpx build --- .github/workflows/release.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3eef632..beeed0b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -98,10 +98,14 @@ jobs: set -x git clone --depth 1 https://chromium.googlesource.com/webm/libvpx -b "$LIBVPX_TAG" cd libvpx + # NOTE: disabling unit tests and examples significantly reduces build + # time (by 80% as tested on a Jetson Nano) ./configure \ --enable-vp8 \ --enable-vp9 \ --enable-runtime-cpu-detect \ + --disable-unit-tests \ + --disable-examples \ --enable-static \ --disable-shared make From 276ee197f206b1d21612e5883a3e9943e969b9be Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 11:48:54 -0700 Subject: [PATCH 072/181] Try disabling all asm on macOS Disabling "x86asm" is not enough, because it still fails to build features controlled by "inline-asm", so add --disable-inline-asm on macOS. --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index beeed0b..c273632 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -215,7 +215,7 @@ jobs: # others are not having this problem with ffmpeg. # TODO: Try building from master branch to see if this has been # resolved more recently than n4.4. - PLATFORM_CONFIGURE_FLAGS="$PLATFORM_CONFIGURE_FLAGS --disable-x86asm" + PLATFORM_CONFIGURE_FLAGS="$PLATFORM_CONFIGURE_FLAGS --disable-x86asm --disable-inline-asm" fi ./configure \ --pkg-config-flags="--static" \ From 4e17bbb8535ce07c86de14dd53e6504773f1633d Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 11:57:52 -0700 Subject: [PATCH 073/181] First attempt at a Windows build Set an environment variable for sudo ($SUDO), which will be empty on Windows. Not sure what else I will need to change, but hopefully I can build all of these things in bash instead of having to resort to separate build instructions for Windows. Also installs nasm & yasm via chocolatey. There may be other things missing, but we'll start with that. --- .github/workflows/release.yaml | 41 +++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c273632..6d5e51b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -41,11 +41,13 @@ jobs: needs: draft_release strategy: matrix: - # TODO: Windows - # TODO: Mac - # TODO: arm64 + # TODO: Work through Windows issues + # TODO: Finish Mac + # TODO: Add linux-arm64 + # TODO: Add Mac arm64? + # TODO: Add Windows arm64? #os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] - os: ["ubuntu-latest", "macos-latest"] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] include: - os: ubuntu-latest os_name: linux @@ -55,6 +57,10 @@ jobs: os_name: osx target_arch: x64 exe_ext: "" + - os: windows-latest + os_name: win + target_arch: x64 + exe_ext: ".exe" name: Build ${{ matrix.os_name }} ${{ matrix.target_arch }} runs-on: ${{ matrix.os }} @@ -71,6 +77,8 @@ jobs: yasm \ libffmpeg-nvenc-dev \ libvdpau-dev + # Use sudo in install commands on Linux. + echo "SUDO=sudo" >> $GITHUB_ENV - name: Install macOS packages if: runner.os == 'macOS' @@ -82,6 +90,8 @@ jobs: mercurial \ nasm \ yasm + # Use sudo in install commands on macOS. + echo "SUDO=sudo" >> $GITHUB_ENV - name: Install Windows packages if: runner.os == 'Windows' @@ -89,7 +99,12 @@ jobs: run: | set -e set -x - true # FIXME: install prereqs & tools on Windows + # TODO: Discover all the missing packages on Windows, flesh this out + choco install \ + nasm \ + yasm + # Don't use sudo in install commands on Windows. + echo "SUDO=" >> $GITHUB_ENV - name: Install libvpx shell: bash @@ -109,7 +124,7 @@ jobs: --enable-static \ --disable-shared make - sudo make install + $SUDO make install - name: Install aom shell: bash @@ -128,7 +143,7 @@ jobs: -DCONFIG_RUNTIME_CPU_DETECT=1 \ -DCONFIG_SHARED=0 make - sudo make install + $SUDO make install - name: Install x264 shell: bash @@ -140,7 +155,7 @@ jobs: ./configure \ --enable-static make - sudo make install + $SUDO make install - name: Install x265 shell: bash @@ -151,11 +166,11 @@ jobs: cd x265/build cmake ../source make - sudo make install + $SUDO make install # This adjustment to the x265 linker flags is needed, at least on # arm, to successfully link against it statically. (-lgcc_s not # found (or needed), and -lpthread missing) - sudo sed -e 's/-lgcc_s -lgcc -lgcc_s -lgcc/-lpthread -lgcc/' -i.bk /usr/local/lib/pkgconfig/x265.pc + $SUDO sed -e 's/-lgcc_s -lgcc -lgcc_s -lgcc/-lpthread -lgcc/' -i.bk /usr/local/lib/pkgconfig/x265.pc - name: Install lame shell: bash @@ -169,7 +184,7 @@ jobs: --enable-static \ --disable-shared make - sudo make install + $SUDO make install - name: Install opus shell: bash @@ -183,10 +198,10 @@ jobs: --enable-static \ --disable-shared make - sudo make install + $SUDO make install # The pkgconfig linker flags for static opus don't work when ffmpeg # checks for opus in configure. Linking libm after libopus fixes it. - sudo sed -e 's/-lopus/-lopus -lm/' -i.bk /usr/local/lib/pkgconfig/opus.pc + $SUDO sed -e 's/-lopus/-lopus -lm/' -i.bk /usr/local/lib/pkgconfig/opus.pc - name: Build ffmpeg and ffprobe shell: bash From 3d9c30ad6ab14a684ac6f28a53e9dbd1a1be5460 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 12:03:36 -0700 Subject: [PATCH 074/181] Fix yasm install on Windows --- .github/workflows/release.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6d5e51b..e5dfae1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -100,7 +100,9 @@ jobs: set -e set -x # TODO: Discover all the missing packages on Windows, flesh this out + # NOTE: yasm does not have checksums in the choco package. choco install \ + --allow-empty-checksums \ nasm \ yasm # Don't use sudo in install commands on Windows. From ad75d66495b4e2a8d8bbde153bb601fd91c0bf95 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 12:32:00 -0700 Subject: [PATCH 075/181] Install md5sum on macos --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e5dfae1..3c68b13 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -87,6 +87,7 @@ jobs: set -e set -x brew install \ + md5sha1sum \ mercurial \ nasm \ yasm From c1a1b45b347f41e48e8b4f318f2137313caea4b0 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 16:48:07 -0700 Subject: [PATCH 076/181] Install gnu make on Windows, adjust ldd usage on all platforms --- .github/workflows/release.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3c68b13..1ef0ddc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -104,6 +104,7 @@ jobs: # NOTE: yasm does not have checksums in the choco package. choco install \ --allow-empty-checksums \ + make \ nasm \ yasm # Don't use sudo in install commands on Windows. @@ -251,8 +252,17 @@ jobs: $PLATFORM_CONFIGURE_FLAGS make # Show that these are not dynamic executables. Fail if they are. - ldd ffmpeg && exit 1 - ldd ffprobe && exit 1 + if [[ "${{ runner.os }}" == "Linux" ]]; then + ldd ffmpeg && exit 1 + ldd ffprobe && exit 1 + elif [[ "${{ runner.os }}" == "macOS" ]]; then + # TODO: Verify the usage of otool for macOS. + otool -L ffmpeg && exit 1 + otool -L ffprobe && exit 1 + elif [[ "${{ runner.os }}" == "Windows" ]]; then + ldd ffmpeg.exe && exit 1 + ldd ffprobe.exe && exit 1 + fi # After commands that we expect to fail, we still need a successful # command here to make this step a success. true From 7e78d00ca48079140fe92720d562ad754dcbcade Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 16:56:19 -0700 Subject: [PATCH 077/181] Debugging make failures on Windows --- .github/workflows/release.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1ef0ddc..c3055f9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -46,8 +46,7 @@ jobs: # TODO: Add linux-arm64 # TODO: Add Mac arm64? # TODO: Add Windows arm64? - #os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] - os: ["ubuntu-latest", "macos-latest", "windows-latest"] + os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] include: - os: ubuntu-latest os_name: linux @@ -61,6 +60,10 @@ jobs: os_name: win target_arch: x64 exe_ext: ".exe" + - os: linux-arm64 + os_name: linux + target_arch: arm64 + exe_ext: "" name: Build ${{ matrix.os_name }} ${{ matrix.target_arch }} runs-on: ${{ matrix.os }} @@ -109,6 +112,14 @@ jobs: yasm # Don't use sudo in install commands on Windows. echo "SUDO=" >> $GITHUB_ENV + # FIXME: Debugging + # In the libvpx build, make doesn't seem to work. If I SSH into the + # VM, make does work. So am I getting a different make? How is the + # build environment different from the shell I get when I ssh in? + echo "PATH=$PATH" + which make + PATH="/usr/bin:$PATH" which make + echo "/usr/bin" >> $GITHUB_PATH - name: Install libvpx shell: bash From f8e035fd0de6444d15ab5aaa2b116ec8fce2b2d9 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 17:01:50 -0700 Subject: [PATCH 078/181] More make debugging on Windows --- .github/workflows/release.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c3055f9..d870629 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -43,7 +43,6 @@ jobs: matrix: # TODO: Work through Windows issues # TODO: Finish Mac - # TODO: Add linux-arm64 # TODO: Add Mac arm64? # TODO: Add Windows arm64? os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] @@ -107,7 +106,6 @@ jobs: # NOTE: yasm does not have checksums in the choco package. choco install \ --allow-empty-checksums \ - make \ nasm \ yasm # Don't use sudo in install commands on Windows. @@ -117,9 +115,13 @@ jobs: # VM, make does work. So am I getting a different make? How is the # build environment different from the shell I get when I ssh in? echo "PATH=$PATH" - which make - PATH="/usr/bin:$PATH" which make - echo "/usr/bin" >> $GITHUB_PATH + # Show all make options + which -a make + # Are these specific ones here? They are via SSH later. + ls /usr/bin/make + ls /usr/bin/make.exe + ls /bin/make + ls /bin/make.exe - name: Install libvpx shell: bash From e3e386c9cadc7f0f2e1fb5f89685840385ac6586 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sat, 7 Aug 2021 17:11:25 -0700 Subject: [PATCH 079/181] Try msys2 to get a working "make" on Windows --- .github/workflows/release.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d870629..6b82f07 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -104,8 +104,13 @@ jobs: set -x # TODO: Discover all the missing packages on Windows, flesh this out # NOTE: yasm does not have checksums in the choco package. + # NOTE: msys2 gives us a working version of gnu make. The version + # already installed (choco package "make") somehow fails to build the + # libraries below. + # TODO: Verify my suspicions about msys2 and make. choco install \ --allow-empty-checksums \ + msys2 \ nasm \ yasm # Don't use sudo in install commands on Windows. @@ -114,14 +119,11 @@ jobs: # In the libvpx build, make doesn't seem to work. If I SSH into the # VM, make does work. So am I getting a different make? How is the # build environment different from the shell I get when I ssh in? + # It seems that action-tmate installs msys, so that may be the key. + # The working version is for msys, vs mingw. echo "PATH=$PATH" # Show all make options which -a make - # Are these specific ones here? They are via SSH later. - ls /usr/bin/make - ls /usr/bin/make.exe - ls /bin/make - ls /bin/make.exe - name: Install libvpx shell: bash From 78ec0907d0da341794115b989ac1f355837b8daf Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 16:32:54 -0700 Subject: [PATCH 080/181] Try installing make via pacman on Windows --- .github/workflows/release.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6b82f07..47190be 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -104,17 +104,17 @@ jobs: set -x # TODO: Discover all the missing packages on Windows, flesh this out # NOTE: yasm does not have checksums in the choco package. - # NOTE: msys2 gives us a working version of gnu make. The version - # already installed (choco package "make") somehow fails to build the - # libraries below. - # TODO: Verify my suspicions about msys2 and make. choco install \ --allow-empty-checksums \ - msys2 \ nasm \ yasm # Don't use sudo in install commands on Windows. echo "SUDO=" >> $GITHUB_ENV + # NOTE: msys2 gives us a working version of gnu make. The version + # already installed (choco package "make") somehow fails to build the + # libraries below. + # TODO: Verify my suspicions about msys2 and make. + pacman -S mingw-w64-x86_64-make # FIXME: Debugging # In the libvpx build, make doesn't seem to work. If I SSH into the # VM, make does work. So am I getting a different make? How is the From b1d98303effd8c1920cd39aa4529af2d72d984e9 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 17:48:06 -0700 Subject: [PATCH 081/181] Install cmake on Linux to fix arm64 build The self-hosted runner images we use for arm64 do not contain cmake. --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 47190be..3ce972e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -75,6 +75,7 @@ jobs: set -e set -x sudo apt -y install \ + cmake \ nasm \ yasm \ libffmpeg-nvenc-dev \ From f51edaebc1d6f0d07e2bd35b430f2944c13732ec Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 17:48:17 -0700 Subject: [PATCH 082/181] More windows "make" debugging --- .github/workflows/release.yaml | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3ce972e..46ba068 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -109,22 +109,18 @@ jobs: --allow-empty-checksums \ nasm \ yasm - # Don't use sudo in install commands on Windows. - echo "SUDO=" >> $GITHUB_ENV # NOTE: msys2 gives us a working version of gnu make. The version - # already installed (choco package "make") somehow fails to build the - # libraries below. - # TODO: Verify my suspicions about msys2 and make. - pacman -S mingw-w64-x86_64-make + # of "make" already in the PATH by default (choco package "make") + # somehow fails to build the libraries below. # FIXME: Debugging - # In the libvpx build, make doesn't seem to work. If I SSH into the - # VM, make does work. So am I getting a different make? How is the - # build environment different from the shell I get when I ssh in? - # It seems that action-tmate installs msys, so that may be the key. - # The working version is for msys, vs mingw. echo "PATH=$PATH" - # Show all make options - which -a make + echo "SHELL=$SHELL" + ls /c/msys64 || true + ls /c/tools/msys64 || true + find /c/ -iname 'make*.exe' -or -iname '*make.exe' || true + find /d/ -iname 'make*.exe' -or -iname '*make.exe' || true + # Don't use sudo in install commands on Windows. + echo "SUDO=" >> $GITHUB_ENV - name: Install libvpx shell: bash From d3a78d3439457af3b0fcd1b9889e23d204ece355 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 18:13:49 -0700 Subject: [PATCH 083/181] Install mercurial on Linux to fix arm64 build The self-hosted runner images we use for arm64 do not contain mercurial. --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 46ba068..06a4c1c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -76,6 +76,7 @@ jobs: set -x sudo apt -y install \ cmake \ + mercurial \ nasm \ yasm \ libffmpeg-nvenc-dev \ From 85b007ba15ee8d7d2deda82837b2706a77b4b251 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 18:55:49 -0700 Subject: [PATCH 084/181] Disable linux-arm64 While we are working through Windows issues, linux-arm64 failures are making debugging on Windows more difficult. --- .github/workflows/release.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 06a4c1c..e234ef2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -43,9 +43,11 @@ jobs: matrix: # TODO: Work through Windows issues # TODO: Finish Mac + # TODO: Work through Linux-arm64 issues # TODO: Add Mac arm64? # TODO: Add Windows arm64? - os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] + #os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] + os: ["ubuntu-latest", "macos-latest", "windows-latest"] include: - os: ubuntu-latest os_name: linux @@ -59,10 +61,10 @@ jobs: os_name: win target_arch: x64 exe_ext: ".exe" - - os: linux-arm64 - os_name: linux - target_arch: arm64 - exe_ext: "" + #- os: linux-arm64 + # os_name: linux + # target_arch: arm64 + # exe_ext: "" name: Build ${{ matrix.os_name }} ${{ matrix.target_arch }} runs-on: ${{ matrix.os }} From be02033ba9e2f0e078630d83a944f6c00fb50b71 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 20:01:14 -0700 Subject: [PATCH 085/181] Install pkg-config on Linux to fix arm64 build The self-hosted runner images we use for arm64 do not contain pkg-config. --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e234ef2..b591215 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -80,6 +80,7 @@ jobs: cmake \ mercurial \ nasm \ + pkg-config \ yasm \ libffmpeg-nvenc-dev \ libvdpau-dev From b33c77ee0721028881e7d8e3faa9b4f318d91c78 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 20:01:37 -0700 Subject: [PATCH 086/181] Fix aom build on arm64 Another linker flag adjustment was needed. --- .github/workflows/release.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b591215..cd88c75 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -164,6 +164,9 @@ jobs: -DCONFIG_SHARED=0 make $SUDO make install + # This adjustment to the aom linker flags is needed, at least on + # arm, to successfully link against it statically. (-lm missing) + $SUDO sed -e 's/-laom/-laom -lm/' -i.bk /usr/local/lib/pkgconfig/aom.pc - name: Install x264 shell: bash From 54966d208b887658a5624cd7beec6de8bb142fc5 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 20:01:54 -0700 Subject: [PATCH 087/181] Fix "make" on Windows I finally found the path to the correct, working version of "make" in the default GitHub Actions Windows image. --- .github/workflows/release.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index cd88c75..97cb4e7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -115,14 +115,9 @@ jobs: yasm # NOTE: msys2 gives us a working version of gnu make. The version # of "make" already in the PATH by default (choco package "make") - # somehow fails to build the libraries below. - # FIXME: Debugging - echo "PATH=$PATH" - echo "SHELL=$SHELL" - ls /c/msys64 || true - ls /c/tools/msys64 || true - find /c/ -iname 'make*.exe' -or -iname '*make.exe' || true - find /d/ -iname 'make*.exe' -or -iname '*make.exe' || true + # somehow fails to build the libraries below. So add the msys + # version to the PATH. + echo "/c/msys64/usr/bin" >> $GITHUB_PATH # Don't use sudo in install commands on Windows. echo "SUDO=" >> $GITHUB_ENV From 3656b011cea3c462eeb53ba7f9d302383e26e456 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 20:03:32 -0700 Subject: [PATCH 088/181] Disable mac Mac binaries are not correct yet. Dynamic, preinstalled versions of libopus, libmp3lame, and others are being linked in to the not-quite-static binaries. For now, while we finish Windows and arm issues, disable mac. --- .github/workflows/release.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 97cb4e7..3185b40 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -47,16 +47,16 @@ jobs: # TODO: Add Mac arm64? # TODO: Add Windows arm64? #os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] - os: ["ubuntu-latest", "macos-latest", "windows-latest"] + os: ["ubuntu-latest", "windows-latest"] include: - os: ubuntu-latest os_name: linux target_arch: x64 exe_ext: "" - - os: macos-latest - os_name: osx - target_arch: x64 - exe_ext: "" + #- os: macos-latest + # os_name: osx + # target_arch: x64 + # exe_ext: "" - os: windows-latest os_name: win target_arch: x64 From 471c04e7cfd0c73abc475bbab7793d4a4e1e6568 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 20:28:05 -0700 Subject: [PATCH 089/181] Try another fix for Windows make So sick of this problem. --- .github/workflows/release.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3185b40..7e40fce 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -113,11 +113,17 @@ jobs: --allow-empty-checksums \ nasm \ yasm - # NOTE: msys2 gives us a working version of gnu make. The version - # of "make" already in the PATH by default (choco package "make") - # somehow fails to build the libraries below. So add the msys - # version to the PATH. - echo "/c/msys64/usr/bin" >> $GITHUB_PATH + # TODO: Debugging "make" failures again. Running the msys make fails + # with a message about a mismatch in the cygwin DLL, so the bash + # shell must use a different DLL than msys does. + # So let's try this to diable "path conversions", which might make + # the default "make" work correctly. No clue. + echo "MSYS_NO_PATHCONV=1" >> $GITHUB_ENV + # While we're at it, let's print the shell version, in case it has + # clues about the specific environment (is it git bash?), and let's + # also print the full windows path of that shell. + $SHELL --version + cygpath -w $SHELL # Don't use sudo in install commands on Windows. echo "SUDO=" >> $GITHUB_ENV From e84e173a2953e10f7c4309731699519d1377bffc Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 20:55:08 -0700 Subject: [PATCH 090/181] Try using the official msys2 action for Windows --- .github/workflows/release.yaml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7e40fce..ce6575e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -101,6 +101,10 @@ jobs: # Use sudo in install commands on macOS. echo "SUDO=sudo" >> $GITHUB_ENV + - name: Setup MSYS2 + if: runner.os == 'Windows' + uses: msys2/setup-msys2@v2 + - name: Install Windows packages if: runner.os == 'Windows' shell: bash @@ -113,17 +117,6 @@ jobs: --allow-empty-checksums \ nasm \ yasm - # TODO: Debugging "make" failures again. Running the msys make fails - # with a message about a mismatch in the cygwin DLL, so the bash - # shell must use a different DLL than msys does. - # So let's try this to diable "path conversions", which might make - # the default "make" work correctly. No clue. - echo "MSYS_NO_PATHCONV=1" >> $GITHUB_ENV - # While we're at it, let's print the shell version, in case it has - # clues about the specific environment (is it git bash?), and let's - # also print the full windows path of that shell. - $SHELL --version - cygpath -w $SHELL # Don't use sudo in install commands on Windows. echo "SUDO=" >> $GITHUB_ENV From 90fd702fdb8f9a923368d5b8e8df48fb349ca70e Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 21:08:39 -0700 Subject: [PATCH 091/181] Try msys as the shell What would happen on Linux? Let's find out! --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ce6575e..c5929d4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -121,7 +121,7 @@ jobs: echo "SUDO=" >> $GITHUB_ENV - name: Install libvpx - shell: bash + shell: msys2 {0} run: | set -e set -x From 5415c79a99c4551ea139b129c4858761232d908c Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 21:21:08 -0700 Subject: [PATCH 092/181] Create compatibility symlink for msys2 Now "msys2" can be the default shell for steps on all platforms, and outside Windows, it will just be /bin/bash. Also, install git on msys. --- .github/workflows/release.yaml | 41 ++++++++++++++-------------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c5929d4..70eb1a2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -69,6 +69,10 @@ jobs: name: Build ${{ matrix.os_name }} ${{ matrix.target_arch }} runs-on: ${{ matrix.os }} + defaults: + run: + shell: msys2 {0} + steps: - name: Install Linux packages if: runner.os == 'Linux' @@ -86,6 +90,9 @@ jobs: libvdpau-dev # Use sudo in install commands on Linux. echo "SUDO=sudo" >> $GITHUB_ENV + # Create a symlink to msys2 so that we can use that as the default + # shell for all platforms. + ln -s /bin/bash /bin/msys2 - name: Install macOS packages if: runner.os == 'macOS' @@ -100,28 +107,22 @@ jobs: yasm # Use sudo in install commands on macOS. echo "SUDO=sudo" >> $GITHUB_ENV - - - name: Setup MSYS2 - if: runner.os == 'Windows' - uses: msys2/setup-msys2@v2 + # Create a symlink to msys2 so that we can use that as the default + # shell for all platforms. + ln -s /bin/bash /bin/msys2 - name: Install Windows packages if: runner.os == 'Windows' - shell: bash - run: | - set -e - set -x - # TODO: Discover all the missing packages on Windows, flesh this out - # NOTE: yasm does not have checksums in the choco package. - choco install \ - --allow-empty-checksums \ - nasm \ + uses: msys2/setup-msys2@v2 + with: + update: true + install: >- + base-devel + git + nasm yasm - # Don't use sudo in install commands on Windows. - echo "SUDO=" >> $GITHUB_ENV - name: Install libvpx - shell: msys2 {0} run: | set -e set -x @@ -141,7 +142,6 @@ jobs: $SUDO make install - name: Install aom - shell: bash run: | set -e set -x @@ -163,7 +163,6 @@ jobs: $SUDO sed -e 's/-laom/-laom -lm/' -i.bk /usr/local/lib/pkgconfig/aom.pc - name: Install x264 - shell: bash run: | set -e set -x @@ -175,7 +174,6 @@ jobs: $SUDO make install - name: Install x265 - shell: bash run: | set -e set -x @@ -190,7 +188,6 @@ jobs: $SUDO sed -e 's/-lgcc_s -lgcc -lgcc_s -lgcc/-lpthread -lgcc/' -i.bk /usr/local/lib/pkgconfig/x265.pc - name: Install lame - shell: bash run: | set -e set -x @@ -204,7 +201,6 @@ jobs: $SUDO make install - name: Install opus - shell: bash run: | set -e set -x @@ -221,7 +217,6 @@ jobs: $SUDO sed -e 's/-lopus/-lopus -lm/' -i.bk /usr/local/lib/pkgconfig/opus.pc - name: Build ffmpeg and ffprobe - shell: bash run: | set -e set -x @@ -287,7 +282,6 @@ jobs: if: failure() - name: Prepare assets - shell: bash run: | set -e set -x @@ -301,7 +295,6 @@ jobs: md5sum * - name: Checkout repo - shell: bash run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - name: Attach assets to release From 7f52298957c3f65cbe3dd839e89c34e21f507bd1 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 21:23:31 -0700 Subject: [PATCH 093/181] Add missing "sudo" --- .github/workflows/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 70eb1a2..13580e8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -92,7 +92,7 @@ jobs: echo "SUDO=sudo" >> $GITHUB_ENV # Create a symlink to msys2 so that we can use that as the default # shell for all platforms. - ln -s /bin/bash /bin/msys2 + sudo ln -s /bin/bash /bin/msys2 - name: Install macOS packages if: runner.os == 'macOS' @@ -109,7 +109,7 @@ jobs: echo "SUDO=sudo" >> $GITHUB_ENV # Create a symlink to msys2 so that we can use that as the default # shell for all platforms. - ln -s /bin/bash /bin/msys2 + sudo ln -s /bin/bash /bin/msys2 - name: Install Windows packages if: runner.os == 'Windows' From 71a7d02f2a6f772905b68959803db3fe5ad5274d Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 21:28:51 -0700 Subject: [PATCH 094/181] Add gcc to msys environment --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 13580e8..6c4dd99 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -118,6 +118,7 @@ jobs: update: true install: >- base-devel + gcc git nasm yasm From e148df5815293d44fe66f0c7f3c30d6d2e75b7ee Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 21:34:39 -0700 Subject: [PATCH 095/181] Add mercurial and cmake to msys environment --- .github/workflows/release.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6c4dd99..7ad7dbd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -118,8 +118,10 @@ jobs: update: true install: >- base-devel + cmake gcc git + mercurial nasm yasm From 44966622b9a4ab6bda0cb3d3e6ab0a993c7280ce Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 22:03:06 -0700 Subject: [PATCH 096/181] Install x264 lib, not exes The exe does not seem to link in msys environment, but we do not need it anyway. --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7ad7dbd..f417586 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -174,7 +174,7 @@ jobs: ./configure \ --enable-static make - $SUDO make install + $SUDO make install install-lib-static - name: Install x265 run: | From 0a0a4f0d00b2b98d36c4b41fcc0b748eedc9d023 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Sun, 8 Aug 2021 22:33:03 -0700 Subject: [PATCH 097/181] Do not build x264 executables --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f417586..8aacf0a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -173,7 +173,7 @@ jobs: cd x264 ./configure \ --enable-static - make + make libx264.a $SUDO make install install-lib-static - name: Install x265 From e687a2608bb9b0feccc5f57941aae4d304d7b50c Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 09:03:31 -0700 Subject: [PATCH 098/181] Fix lib-only x264 install, install x265 lib-only, too --- .github/workflows/release.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8aacf0a..2b1e4e8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -174,7 +174,7 @@ jobs: ./configure \ --enable-static make libx264.a - $SUDO make install install-lib-static + $SUDO make install-lib-static - name: Install x265 run: | @@ -182,7 +182,8 @@ jobs: set -x hg clone http://hg.videolan.org/x265 -r "$X265_TAG" cd x265/build - cmake ../source + cmake ../source \ + -DENABLE_CLI=OFF make $SUDO make install # This adjustment to the x265 linker flags is needed, at least on From 31c2bd150fad943ee6ce698e711cffae10827a49 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 09:07:15 -0700 Subject: [PATCH 099/181] Log asset sizes as well as MD5 sums --- .github/workflows/release.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2b1e4e8..39db4a4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -293,9 +293,10 @@ jobs: SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}${{ matrix.exe_ext}}" cp ffmpeg/ffmpeg assets/ffmpeg"$SUFFIX" cp ffmpeg/ffprobe assets/ffprobe"$SUFFIX" - # Show MD5 sums that can be verified by users later if they want to - # check for authenticity. + # Show sizes and MD5 sums that can be verified by users later if they + # want to check for authenticity. cd assets + wc -c * md5sum * - name: Checkout repo From f79233e6825a466656a695373b4e69a9389c97e7 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 09:29:48 -0700 Subject: [PATCH 100/181] Use mingw compiler on Windows The msys compiler wants to link to a cygwin-like DLL, so use the mingw (native Windows) gcc port instead. This may fix linker errors. --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 39db4a4..352ede1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -119,9 +119,9 @@ jobs: install: >- base-devel cmake - gcc git mercurial + mingw-w64-gcc nasm yasm From d4d61fa4919cae88c205c0b15e8ef1bc20c25ce4 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 09:34:18 -0700 Subject: [PATCH 101/181] Correct msys mingw gcc package name --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 352ede1..43bd93f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -121,7 +121,7 @@ jobs: cmake git mercurial - mingw-w64-gcc + mingw-w64-x86_64-gcc nasm yasm From 7ffdd2d45678e11f2eb356b2a5aa6019571b85db Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 10:32:43 -0700 Subject: [PATCH 102/181] Switch to a fork of action-tmate for SSH debugging This fork is meant to add arm64 support --- .github/workflows/release.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 43bd93f..4e146c5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -279,8 +279,10 @@ jobs: # command here to make this step a success. true + # TODO: send PR for arm64 and unfork + # TODO: remove once the workflow is working correctly - name: Debug - uses: mxschmitt/action-tmate@v3 + uses: joeyparrish/action-tmate@v4 with: limit-access-to-actor: true if: failure() From 17c98a491d88ab31e08881a5bd83a8cad3849158 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 10:34:33 -0700 Subject: [PATCH 103/181] Disable x265 shared library build --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4e146c5..9980d8b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -183,6 +183,7 @@ jobs: hg clone http://hg.videolan.org/x265 -r "$X265_TAG" cd x265/build cmake ../source \ + -DENABLE_SHARED=OFF \ -DENABLE_CLI=OFF make $SUDO make install From 0f9873dd886ae30e59b238a651387d4b93a8792b Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 11:07:58 -0700 Subject: [PATCH 104/181] Install x265 manually This should fix "make install" failures for x265 on Windows and still work everywhere else, too. --- .github/workflows/release.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9980d8b..d941d27 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -186,7 +186,15 @@ jobs: -DENABLE_SHARED=OFF \ -DENABLE_CLI=OFF make - $SUDO make install + # For some reason, make install isn't working for this project on + # Windows. I don't know whether to x265's build system or the crazy + # Actions Windows build environment. Either way, these generic + # instructions work for all platforms. + $SUDO mkdir -p /usr/local/include + $SUDO mkdir -p /usr/local/lib/pkgconfig + $SUDO cp x265_config.h ../source/x265.h /usr/local/include/ + $SUDO cp libx265.a /usr/local/lib/ + $SUDO cp x265.pc /usr/local/lib/pkgconfig/ # This adjustment to the x265 linker flags is needed, at least on # arm, to successfully link against it statically. (-lgcc_s not # found (or needed), and -lpthread missing) From e339c5efccf93a62d1468e66e113b4cc80811b79 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 11:36:59 -0700 Subject: [PATCH 105/181] Re-enable linux-arm64 --- .github/workflows/release.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d941d27..410dffd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -47,7 +47,7 @@ jobs: # TODO: Add Mac arm64? # TODO: Add Windows arm64? #os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] - os: ["ubuntu-latest", "windows-latest"] + os: ["ubuntu-latest", "windows-latest", "linux-arm64"] include: - os: ubuntu-latest os_name: linux @@ -61,10 +61,10 @@ jobs: os_name: win target_arch: x64 exe_ext: ".exe" - #- os: linux-arm64 - # os_name: linux - # target_arch: arm64 - # exe_ext: "" + - os: linux-arm64 + os_name: linux + target_arch: arm64 + exe_ext: "" name: Build ${{ matrix.os_name }} ${{ matrix.target_arch }} runs-on: ${{ matrix.os }} From d97ae393a8c23401d13b081bf27dfd09f2e43105 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 11:37:19 -0700 Subject: [PATCH 106/181] Skip building lame frontend It does not want to build on Windows, and we do not need it anyway. --- .github/workflows/release.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 410dffd..c03d01e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -210,6 +210,9 @@ jobs: ./configure \ --enable-static \ --disable-shared + # Only build and install the library. The frontend doesn't build on + # Windows, and we don't need it anyway. + cd libmp3lame make $SUDO make install From e8a0a6dadc1cf84a61361fbe707aa5029dbffae8 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 12:03:12 -0700 Subject: [PATCH 107/181] Disable lame frontend and opus frontend through configure As opposed to only running "make" in the libmp3lame folder, which does not work when you have not run "make" in other folders it depends on. Also disables the opus frontend. --- .github/workflows/release.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c03d01e..ff9b5a7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -207,12 +207,12 @@ jobs: curl -L -o lame-"$LAME_VERSION".tar.gz https://sourceforge.net/projects/lame/files/lame/"$LAME_VERSION"/lame-"$LAME_VERSION".tar.gz/download tar xzf lame-"$LAME_VERSION".tar.gz cd lame-"$LAME_VERSION" + # Only build and install the library. The frontend doesn't build on + # Windows, and we don't need it anyway. ./configure \ + --disable-frontend \ --enable-static \ --disable-shared - # Only build and install the library. The frontend doesn't build on - # Windows, and we don't need it anyway. - cd libmp3lame make $SUDO make install @@ -224,6 +224,7 @@ jobs: tar xzf opus-"$OPUS_VERSION".tar.gz cd opus-"$OPUS_VERSION" ./configure \ + --disable-extra-programs \ --enable-static \ --disable-shared make From 301f26b10c3530e8132761c79ade9aa06751a2cd Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 12:55:58 -0700 Subject: [PATCH 108/181] Fix opus.pc location on Windows --- .github/workflows/release.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ff9b5a7..770ca93 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -229,6 +229,11 @@ jobs: --disable-shared make $SUDO make install + if [[ "${{ runner.os }}" == "Windows" ]]; then + # On Windows, this one gets installed to a different location. + # Make a symlink. + ln -s /mingw64/lib/pkgconfig/opus.pc /usr/local/lib/pkgconfig/opus.pc + fi # The pkgconfig linker flags for static opus don't work when ffmpeg # checks for opus in configure. Linking libm after libopus fixes it. $SUDO sed -e 's/-lopus/-lopus -lm/' -i.bk /usr/local/lib/pkgconfig/opus.pc From 5d6dd2e5e4c67f7dbb40e294e6f4a64173f3dd7b Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 15:21:54 -0700 Subject: [PATCH 109/181] Fix custom action on self-hosted runner We need to specifically request python3 as the shell, not just "python". --- .github/workflows/custom-github-repo-api-action/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index c9bdc94..d69e78b 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -25,7 +25,7 @@ runs: steps: - name: Call Repo API id: call-repo-api - shell: python + shell: python3 run: | import base64 import json From 736314a6357483b629a4c31e996df50092217c3c Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 15:33:36 -0700 Subject: [PATCH 110/181] Symlink the entire pkgconfig folder on Windows Instead of symlinking the pc file for opus, symlink the entire pkgconfig folder. This way, whichever path the libraries install to will work. Before this, ffmpeg could not find aom on Windows. --- .github/workflows/release.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 770ca93..3ef4efe 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -125,6 +125,17 @@ jobs: nasm yasm + - name: Symlink pkgconfig folder for Windows + if: runner.os == 'Windows' + run: | + set -e + set -x + # On Windows, pkgconfig files go to a different location. + # Make a symlink so that the individual installs below don't need to + # be adjusted. + mkdir -p /usr/local/lib + ln -s /mingw64/lib/pkgconfig /usr/local/lib/pkgconfig + - name: Install libvpx run: | set -e @@ -229,11 +240,6 @@ jobs: --disable-shared make $SUDO make install - if [[ "${{ runner.os }}" == "Windows" ]]; then - # On Windows, this one gets installed to a different location. - # Make a symlink. - ln -s /mingw64/lib/pkgconfig/opus.pc /usr/local/lib/pkgconfig/opus.pc - fi # The pkgconfig linker flags for static opus don't work when ffmpeg # checks for opus in configure. Linking libm after libopus fixes it. $SUDO sed -e 's/-lopus/-lopus -lm/' -i.bk /usr/local/lib/pkgconfig/opus.pc From f50494e4d6a60c8c75c41b7cae54cd051c63f769 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 15:35:29 -0700 Subject: [PATCH 111/181] Make another correction to custom action shell (python3) I thought changing "python" to "python3" would correct the original issue, but it did not. Actions complained about how "python3" was not a registered shell, and that I would need to add "{0}" to it to make it work. --- .github/workflows/custom-github-repo-api-action/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index d69e78b..6a5715b 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -25,7 +25,7 @@ runs: steps: - name: Call Repo API id: call-repo-api - shell: python3 + shell: python3 {0} run: | import base64 import json From f315943498376ae241971e2309dd58025dd5ead4 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 16:25:54 -0700 Subject: [PATCH 112/181] Add debug info for Windows environment Some paths I expect to work do not, and when I connect via tmate, something seems to have changed in a way that is hampering debugging. So we will print the full windows paths of several key locations, then when I connect via tmate later, I can compare. That may help me understand what is wrong. --- .github/workflows/release.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3ef4efe..721d225 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -135,6 +135,15 @@ jobs: # be adjusted. mkdir -p /usr/local/lib ln -s /mingw64/lib/pkgconfig /usr/local/lib/pkgconfig + # FIXME: Debugging msys/mingw environment + cygpath -w / + cygpath -w /mingw64 + cygpath -w /usr/local/lib + cygpath -w /usr/local/lib/pkgconfig + ls -al / + ls -al /mingw64 + ls -al /usr/local/lib + ls -al /usr/local/lib/pkgconfig - name: Install libvpx run: | From 6344cb84140a6780ef7ad5cb14a367c33091c51a Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 16:43:05 -0700 Subject: [PATCH 113/181] Split ldd and otool checks into separate step --- .github/workflows/release.yaml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 721d225..a467118 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -296,17 +296,24 @@ jobs: --enable-static \ $PLATFORM_CONFIGURE_FLAGS make - # Show that these are not dynamic executables. Fail if they are. + + - name: Check that executables are static + run: | + set -e + set -x + cd ffmpeg if [[ "${{ runner.os }}" == "Linux" ]]; then + # Show that these are not dynamic executables. Fail if they are. ldd ffmpeg && exit 1 ldd ffprobe && exit 1 + elif [[ "${{ runner.os }}" == "Windows" ]]; then + # Show that these are not dynamic executables. Fail if they are. + ldd ffmpeg.exe && exit 1 + ldd ffprobe.exe && exit 1 elif [[ "${{ runner.os }}" == "macOS" ]]; then # TODO: Verify the usage of otool for macOS. otool -L ffmpeg && exit 1 otool -L ffprobe && exit 1 - elif [[ "${{ runner.os }}" == "Windows" ]]; then - ldd ffmpeg.exe && exit 1 - ldd ffprobe.exe && exit 1 fi # After commands that we expect to fail, we still need a successful # command here to make this step a success. From ed665bf0920de069b11e3aaf616d93357df35490 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 16:43:32 -0700 Subject: [PATCH 114/181] The linux-arm64 build is complete and working --- .github/workflows/release.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a467118..d40c139 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -43,7 +43,6 @@ jobs: matrix: # TODO: Work through Windows issues # TODO: Finish Mac - # TODO: Work through Linux-arm64 issues # TODO: Add Mac arm64? # TODO: Add Windows arm64? #os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] From 7f57df2ec1d89bfbfdabc62d00d0ec21e5cb165d Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 17:02:38 -0700 Subject: [PATCH 115/181] Try again on Windows without msys2/setup-msys2 My last debug session revealed that msys2/setup-msys2 installs to D:/a/_temp/msys/msys64, and skimming the source code of that action revealed that this is a separate install extracted from the default one in C:/msys64. So there is no reason it should fail if we try to use that. At the very least, this could bring the build environment in line with the tmate debugging environment, which would make debugging easier and more useful. Ultimately, I suspect the symlinks I am creating on Windows are not working. But I would like to do another SSH debug session to confirm and to develop another solution. --- .github/workflows/release.yaml | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d40c139..ba66e8e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -112,16 +112,27 @@ jobs: - name: Install Windows packages if: runner.os == 'Windows' - uses: msys2/setup-msys2@v2 - with: - update: true - install: >- - base-devel - cmake - git - mercurial - mingw-w64-x86_64-gcc - nasm + #uses: msys2/setup-msys2@v2 + #with: + # update: true + # install: >- + # base-devel + # cmake + # git + # mercurial + # mingw-w64-x86_64-gcc + # nasm + # yasm + run: | + set -e + set -x + pacman -Sy --noconfirm \ + base-devel \ + cmake \ + git \ + mercurial \ + mingw-w64-x86_64-gcc \ + nasm \ yasm - name: Symlink pkgconfig folder for Windows From b39b5ac768622862d37125a52b2af9ce051517af Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 17:18:05 -0700 Subject: [PATCH 116/181] Try adding the default msys2 to the path Before running other msys2/mingw64 steps on Windows, try adding msys2 to the path. This path may not be correct yet, or there may be others required. --- .github/workflows/release.yaml | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ba66e8e..ddf1675 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -70,7 +70,7 @@ jobs: defaults: run: - shell: msys2 {0} + shell: bash steps: - name: Install Linux packages @@ -89,9 +89,6 @@ jobs: libvdpau-dev # Use sudo in install commands on Linux. echo "SUDO=sudo" >> $GITHUB_ENV - # Create a symlink to msys2 so that we can use that as the default - # shell for all platforms. - sudo ln -s /bin/bash /bin/msys2 - name: Install macOS packages if: runner.os == 'macOS' @@ -106,26 +103,21 @@ jobs: yasm # Use sudo in install commands on macOS. echo "SUDO=sudo" >> $GITHUB_ENV - # Create a symlink to msys2 so that we can use that as the default - # shell for all platforms. - sudo ln -s /bin/bash /bin/msys2 + + - name: Add msys2 to the Windows path + if: runner.os == 'Windows' + run: | + echo "C:\\msys64\\mingw64\\bin" >> "$GITHUB_PATH" + # TODO: Clean up this debug code + cygpath -w "$SHELL" - name: Install Windows packages if: runner.os == 'Windows' - #uses: msys2/setup-msys2@v2 - #with: - # update: true - # install: >- - # base-devel - # cmake - # git - # mercurial - # mingw-w64-x86_64-gcc - # nasm - # yasm run: | set -e set -x + # TODO: Clean up this debug code + cygpath -w "$SHELL" pacman -Sy --noconfirm \ base-devel \ cmake \ From b5d90c137a4dce1a60b626703a03ad7f023ec71f Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 17:23:18 -0700 Subject: [PATCH 117/181] Add another msys path on Windows --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ddf1675..b2e4df3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -108,6 +108,7 @@ jobs: if: runner.os == 'Windows' run: | echo "C:\\msys64\\mingw64\\bin" >> "$GITHUB_PATH" + echo "C:\\msys64\\usr\\bin" >> "$GITHUB_PATH" # TODO: Clean up this debug code cygpath -w "$SHELL" From 2480cbb6cecb245e946d92e18e6fcf00d36afa5f Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 17:49:04 -0700 Subject: [PATCH 118/181] Do not reinstall default Windows packages Some of these turn out to be installed already. When we were installing a fresh system, they needed to be specified. Now specifying them causes them to be redownloaded and reinstalled. Also remove the shell debugging, as this is working correctly now. --- .github/workflows/release.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b2e4df3..12d5a43 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -107,24 +107,20 @@ jobs: - name: Add msys2 to the Windows path if: runner.os == 'Windows' run: | + # At this point, we're running Git Bash. After this step, we will be + # running msys bash, just as we would be in tmate. echo "C:\\msys64\\mingw64\\bin" >> "$GITHUB_PATH" echo "C:\\msys64\\usr\\bin" >> "$GITHUB_PATH" - # TODO: Clean up this debug code - cygpath -w "$SHELL" - name: Install Windows packages if: runner.os == 'Windows' run: | set -e set -x - # TODO: Clean up this debug code - cygpath -w "$SHELL" pacman -Sy --noconfirm \ - base-devel \ cmake \ git \ mercurial \ - mingw-w64-x86_64-gcc \ nasm \ yasm From 4607a0005e81a593c71b92baf531af77f9ee25dc Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 18:26:03 -0700 Subject: [PATCH 119/181] Debugging cmake and symlink issues on Windows The last build failed for lack of a working cmake, but while connected via SSH, I found that we had multiple installs of it. This reverses the order of msys paths, in the hopes that this will fix it. I got disconnected before I could verify that, though. --- .github/workflows/release.yaml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 12d5a43..4b46784 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -109,20 +109,27 @@ jobs: run: | # At this point, we're running Git Bash. After this step, we will be # running msys bash, just as we would be in tmate. - echo "C:\\msys64\\mingw64\\bin" >> "$GITHUB_PATH" + # Make sure we add them in this order, so that we get a functional + # cmake for aom and x265. I don't know why/how we have one in each + # of these paths, but there it is. echo "C:\\msys64\\usr\\bin" >> "$GITHUB_PATH" + echo "C:\\msys64\\mingw64\\bin" >> "$GITHUB_PATH" - name: Install Windows packages if: runner.os == 'Windows' run: | set -e set -x + # FIXME: Debugging cmake issues + which -a cmake pacman -Sy --noconfirm \ cmake \ git \ mercurial \ nasm \ yasm + # FIXME: Debugging cmake issues + which -a cmake - name: Symlink pkgconfig folder for Windows if: runner.os == 'Windows' @@ -136,13 +143,9 @@ jobs: ln -s /mingw64/lib/pkgconfig /usr/local/lib/pkgconfig # FIXME: Debugging msys/mingw environment cygpath -w / - cygpath -w /mingw64 - cygpath -w /usr/local/lib - cygpath -w /usr/local/lib/pkgconfig - ls -al / - ls -al /mingw64 - ls -al /usr/local/lib + ls -al /mingw64/lib/pkgconfig ls -al /usr/local/lib/pkgconfig + ls -ld /usr/local/lib/pkgconfig - name: Install libvpx run: | From e705080b0f958f7bca8addd02947e018dd8b4054 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 19:42:48 -0700 Subject: [PATCH 120/181] Set cmake and pkg-config settings for windows --- .github/workflows/release.yaml | 39 ++++++++++++---------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4b46784..b264562 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -88,7 +88,7 @@ jobs: libffmpeg-nvenc-dev \ libvdpau-dev # Use sudo in install commands on Linux. - echo "SUDO=sudo" >> $GITHUB_ENV + echo "SUDO=sudo" >> "$GITHUB_ENV" - name: Install macOS packages if: runner.os == 'macOS' @@ -102,16 +102,13 @@ jobs: nasm \ yasm # Use sudo in install commands on macOS. - echo "SUDO=sudo" >> $GITHUB_ENV + echo "SUDO=sudo" >> "$GITHUB_ENV" - name: Add msys2 to the Windows path if: runner.os == 'Windows' run: | # At this point, we're running Git Bash. After this step, we will be # running msys bash, just as we would be in tmate. - # Make sure we add them in this order, so that we get a functional - # cmake for aom and x265. I don't know why/how we have one in each - # of these paths, but there it is. echo "C:\\msys64\\usr\\bin" >> "$GITHUB_PATH" echo "C:\\msys64\\mingw64\\bin" >> "$GITHUB_PATH" @@ -120,32 +117,15 @@ jobs: run: | set -e set -x - # FIXME: Debugging cmake issues - which -a cmake pacman -Sy --noconfirm \ - cmake \ git \ mercurial \ nasm \ yasm - # FIXME: Debugging cmake issues - which -a cmake - - - name: Symlink pkgconfig folder for Windows - if: runner.os == 'Windows' - run: | - set -e - set -x - # On Windows, pkgconfig files go to a different location. - # Make a symlink so that the individual installs below don't need to - # be adjusted. - mkdir -p /usr/local/lib - ln -s /mingw64/lib/pkgconfig /usr/local/lib/pkgconfig - # FIXME: Debugging msys/mingw environment - cygpath -w / - ls -al /mingw64/lib/pkgconfig - ls -al /usr/local/lib/pkgconfig - ls -ld /usr/local/lib/pkgconfig + # Make sure that cmake builds makefiles and not ninja files. + echo "CMAKE_GENERATOR=MSYS Makefiles" >> "$GITHUB_ENV" + # Make sure that pkg-config searches the expected path. + echo "PKG_CONFIG_PATH=/usr/local/lib/pkgconfig" >> "$GITHUB_ENV" - name: Install libvpx run: | @@ -173,7 +153,10 @@ jobs: git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" mkdir aom_build cd aom_build + # NOTE: without CMAKE_INSTALL_PREFIX on Windows, files are installed + # to c:\Program Files. cmake ../aom \ + -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_DOCS=OFF \ -DENABLE_EXAMPLES=OFF \ -DENABLE_TESTS=OFF \ @@ -204,10 +187,14 @@ jobs: set -x hg clone http://hg.videolan.org/x265 -r "$X265_TAG" cd x265/build + # NOTE: without CMAKE_INSTALL_PREFIX on Windows, files are installed + # to c:\Program Files. cmake ../source \ + -DCMAKE_INSTALL_PREFIX=/usr/local -DENABLE_SHARED=OFF \ -DENABLE_CLI=OFF make + # TODO: Re-evaluate this now that I have simplified the environment. # For some reason, make install isn't working for this project on # Windows. I don't know whether to x265's build system or the crazy # Actions Windows build environment. Either way, these generic From 4577ef9ce759357a16b8a756e64a35ce48d25284 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 19:51:11 -0700 Subject: [PATCH 121/181] Add missing backslash in cmake commands --- .github/workflows/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b264562..eb36ecc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -156,7 +156,7 @@ jobs: # NOTE: without CMAKE_INSTALL_PREFIX on Windows, files are installed # to c:\Program Files. cmake ../aom \ - -DCMAKE_INSTALL_PREFIX=/usr/local + -DCMAKE_INSTALL_PREFIX=/usr/local \ -DENABLE_DOCS=OFF \ -DENABLE_EXAMPLES=OFF \ -DENABLE_TESTS=OFF \ @@ -190,7 +190,7 @@ jobs: # NOTE: without CMAKE_INSTALL_PREFIX on Windows, files are installed # to c:\Program Files. cmake ../source \ - -DCMAKE_INSTALL_PREFIX=/usr/local + -DCMAKE_INSTALL_PREFIX=/usr/local \ -DENABLE_SHARED=OFF \ -DENABLE_CLI=OFF make From 1f04afd0da8a960f2fa946f55f308086ea37ad91 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 20:37:17 -0700 Subject: [PATCH 122/181] Fix Windows install of opus --- .github/workflows/release.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index eb36ecc..80ba1e0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -232,7 +232,10 @@ jobs: curl -LO https://archive.mozilla.org/pub/opus/opus-"$OPUS_VERSION".tar.gz tar xzf opus-"$OPUS_VERSION".tar.gz cd opus-"$OPUS_VERSION" + # On Windows, somehow prefix defaults to / instead of /usr/local, but + # only on this one project. No idea why. ./configure \ + --prefix=/usr/local \ --disable-extra-programs \ --enable-static \ --disable-shared From 3110463cccbfb249bfc3221e6f79cd6c15949ab5 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 9 Aug 2021 21:22:55 -0700 Subject: [PATCH 123/181] Fix windows install of lame --- .github/workflows/release.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 80ba1e0..1fce9a7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -216,9 +216,12 @@ jobs: curl -L -o lame-"$LAME_VERSION".tar.gz https://sourceforge.net/projects/lame/files/lame/"$LAME_VERSION"/lame-"$LAME_VERSION".tar.gz/download tar xzf lame-"$LAME_VERSION".tar.gz cd lame-"$LAME_VERSION" - # Only build and install the library. The frontend doesn't build on - # Windows, and we don't need it anyway. + # Only build and install the library (--disable-front-end). The + # frontend doesn't build on Windows, and we don't need it anyway. + # On Windows, somehow prefix defaults to / instead of /usr/local, but + # only on some projects. No idea why. ./configure \ + --prefix=/usr/local \ --disable-frontend \ --enable-static \ --disable-shared @@ -233,7 +236,7 @@ jobs: tar xzf opus-"$OPUS_VERSION".tar.gz cd opus-"$OPUS_VERSION" # On Windows, somehow prefix defaults to / instead of /usr/local, but - # only on this one project. No idea why. + # only on some projects. No idea why. ./configure \ --prefix=/usr/local \ --disable-extra-programs \ From 5ec64a731110d25cc25c60657aa2bf5a16b86476 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 07:05:23 -0700 Subject: [PATCH 124/181] Debug platform detection for ffmpeg on Windows The ffmpeg configure script tries to detect what the OS is. It supports mingw, but not msys. The previous build died with "Native MSYS builds are discouraged, please use the MINGW environment". Run uname -s to try to determine what ffmpeg configure will see and why. --- .github/workflows/release.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1fce9a7..ef31a70 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -111,12 +111,20 @@ jobs: # running msys bash, just as we would be in tmate. echo "C:\\msys64\\usr\\bin" >> "$GITHUB_PATH" echo "C:\\msys64\\mingw64\\bin" >> "$GITHUB_PATH" + # TODO: Debug + which -a uname + cygpath -w $(which uname) + uname -s - name: Install Windows packages if: runner.os == 'Windows' run: | set -e set -x + # TODO: Debug + which -a uname + cygpath -w $(which uname) + uname -s pacman -Sy --noconfirm \ git \ mercurial \ @@ -126,6 +134,10 @@ jobs: echo "CMAKE_GENERATOR=MSYS Makefiles" >> "$GITHUB_ENV" # Make sure that pkg-config searches the expected path. echo "PKG_CONFIG_PATH=/usr/local/lib/pkgconfig" >> "$GITHUB_ENV" + # TODO: Debug + which -a uname + cygpath -w $(which uname) + uname -s - name: Install libvpx run: | From 74c3ff80e6fd3e62b7e54569249ac9d3176e9a05 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 07:27:04 -0700 Subject: [PATCH 125/181] Bump tmate fork version for testing on arm64 --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ef31a70..c4e436d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -329,7 +329,7 @@ jobs: # TODO: send PR for arm64 and unfork # TODO: remove once the workflow is working correctly - name: Debug - uses: joeyparrish/action-tmate@v4 + uses: joeyparrish/action-tmate@v5 with: limit-access-to-actor: true if: failure() From ac948842b0f5718b5e9ecd3873e8ad4bf99d91b6 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 07:44:34 -0700 Subject: [PATCH 126/181] Revert "Debug platform detection for ffmpeg on Windows" This reverts commit 5ec64a731110d25cc25c60657aa2bf5a16b86476. --- .github/workflows/release.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c4e436d..b7a3f32 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -111,20 +111,12 @@ jobs: # running msys bash, just as we would be in tmate. echo "C:\\msys64\\usr\\bin" >> "$GITHUB_PATH" echo "C:\\msys64\\mingw64\\bin" >> "$GITHUB_PATH" - # TODO: Debug - which -a uname - cygpath -w $(which uname) - uname -s - name: Install Windows packages if: runner.os == 'Windows' run: | set -e set -x - # TODO: Debug - which -a uname - cygpath -w $(which uname) - uname -s pacman -Sy --noconfirm \ git \ mercurial \ @@ -134,10 +126,6 @@ jobs: echo "CMAKE_GENERATOR=MSYS Makefiles" >> "$GITHUB_ENV" # Make sure that pkg-config searches the expected path. echo "PKG_CONFIG_PATH=/usr/local/lib/pkgconfig" >> "$GITHUB_ENV" - # TODO: Debug - which -a uname - cygpath -w $(which uname) - uname -s - name: Install libvpx run: | From a1855a1c56d58f86848a64513cda9a6ca2c1c06b Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 07:44:29 -0700 Subject: [PATCH 127/181] Specify mingw target for ffmpeg on Windows --- .github/workflows/release.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b7a3f32..7f7a64c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -275,6 +275,12 @@ jobs: # TODO: Try building from master branch to see if this has been # resolved more recently than n4.4. PLATFORM_CONFIGURE_FLAGS="$PLATFORM_CONFIGURE_FLAGS --disable-x86asm --disable-inline-asm" + elif [[ "${{ runner.os }}" == "Windows" ]]; then + # Convince ffmpeg that we want to build for mingw64 (native + # Windows), not msys. Since we're in an msys environment, ffmpeg + # reasonably assumes we're building for that environment if we + # don't specify this. + PLATFORM_CONFIGURE_FLAGS="--target-os=mingw64" fi ./configure \ --pkg-config-flags="--static" \ From 7bea3b9b44aa97194aabb8f5d5725c8188de6855 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 07:55:44 -0700 Subject: [PATCH 128/181] Specify LDFLAGS for ffmpeg on Windows Surprisingly, /usr/local/lib is not in the mingw linker path by default. --- .github/workflows/release.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7f7a64c..f238c9f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -276,6 +276,9 @@ jobs: # resolved more recently than n4.4. PLATFORM_CONFIGURE_FLAGS="$PLATFORM_CONFIGURE_FLAGS --disable-x86asm --disable-inline-asm" elif [[ "${{ runner.os }}" == "Windows" ]]; then + # Surprisingly, this is not in the mingw linker path by default. + export LDFLAGS="-L/usr/local/lib" + # Convince ffmpeg that we want to build for mingw64 (native # Windows), not msys. Since we're in an msys environment, ffmpeg # reasonably assumes we're building for that environment if we From 70b2ff816df98f9f09c5d84dfd6ab17b9c369038 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 08:04:18 -0700 Subject: [PATCH 129/181] Disable stack-protector for opus on Windows Otherwise, we get link failures in ffmpeg about missing stack-protector symbols. --- .github/workflows/release.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f238c9f..289deb5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -237,9 +237,11 @@ jobs: cd opus-"$OPUS_VERSION" # On Windows, somehow prefix defaults to / instead of /usr/local, but # only on some projects. No idea why. + # On Windows, we also need to disable-stack-protector. ./configure \ --prefix=/usr/local \ --disable-extra-programs \ + --disable-stack-protector \ --enable-static \ --disable-shared make From fa2baf9a6e747e50b87aa3ec812b016678efc33b Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 08:24:01 -0700 Subject: [PATCH 130/181] Remove _FORTIFY_SOURCE from opus build CFLAGS This flag makes linking fail on Windows. But there is no configure option for this, so we edit the configure script instead. --- .github/workflows/release.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 289deb5..92deb87 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -235,6 +235,10 @@ jobs: curl -LO https://archive.mozilla.org/pub/opus/opus-"$OPUS_VERSION".tar.gz tar xzf opus-"$OPUS_VERSION".tar.gz cd opus-"$OPUS_VERSION" + # On Windows, we can't link later if we build with -D_FORTIFY_SOURCE + # now. But there is no configure option for this, so we edit the + # configure script instead. + sed -e 's/-D_FORTIFY_SOURCE=2//' -i.bk configure # On Windows, somehow prefix defaults to / instead of /usr/local, but # only on some projects. No idea why. # On Windows, we also need to disable-stack-protector. From 80c6f01c898dca62ab0dd2f9d7f99f8ff7f3c206 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 11:08:05 -0700 Subject: [PATCH 131/181] Try to build ffmpeg with -static on Windows --- .github/workflows/release.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 92deb87..a814501 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -282,8 +282,10 @@ jobs: # resolved more recently than n4.4. PLATFORM_CONFIGURE_FLAGS="$PLATFORM_CONFIGURE_FLAGS --disable-x86asm --disable-inline-asm" elif [[ "${{ runner.os }}" == "Windows" ]]; then - # Surprisingly, this is not in the mingw linker path by default. - export LDFLAGS="-L/usr/local/lib" + export CFLAGS="-static" + # Surprisingly, /usr/local/lib is not in the mingw linker path by + # default. + export LDFLAGS="-static -L/usr/local/lib" # Convince ffmpeg that we want to build for mingw64 (native # Windows), not msys. Since we're in an msys environment, ffmpeg From b98417fd514cb8a1d1429b9ef89acd9d0e696253 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 12:49:28 -0700 Subject: [PATCH 132/181] Adjust expectations for dynamic executables on Windows I think this Windows build will finally pass! --- .github/workflows/release.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a814501..5fdc8f7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -41,7 +41,6 @@ jobs: needs: draft_release strategy: matrix: - # TODO: Work through Windows issues # TODO: Finish Mac # TODO: Add Mac arm64? # TODO: Add Windows arm64? @@ -315,13 +314,17 @@ jobs: set -x cd ffmpeg if [[ "${{ runner.os }}" == "Linux" ]]; then - # Show that these are not dynamic executables. Fail if they are. + # If ldd succeeds, then these are dynamic executables, so we fail + # this step if ldd succeeds. ldd ffmpeg && exit 1 ldd ffprobe && exit 1 elif [[ "${{ runner.os }}" == "Windows" ]]; then - # Show that these are not dynamic executables. Fail if they are. - ldd ffmpeg.exe && exit 1 - ldd ffprobe.exe && exit 1 + # These will still be dynamic executables, but they should not link + # against anything outside of /c/Windows. The grep command will + # succeed if it can find anything outside /c/Windows, and then we + # fail if that succeeds. + ldd ffmpeg.exe | grep -qvi /c/Windows/ && exit 1 + ldd ffprobe.exe | grep -qvi /c/Windows/ && exit 1 elif [[ "${{ runner.os }}" == "macOS" ]]; then # TODO: Verify the usage of otool for macOS. otool -L ffmpeg && exit 1 From 415c047bf8fd0ab197ac8585431cdf9cd5bc254f Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 12:50:41 -0700 Subject: [PATCH 133/181] Re-enable mac build Last I tried on mac, it was creating dynamic executables that linked to non-standard libraries in the GitHub Actions environment. So we still need to address that. --- .github/workflows/release.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5fdc8f7..5b09f20 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -44,17 +44,16 @@ jobs: # TODO: Finish Mac # TODO: Add Mac arm64? # TODO: Add Windows arm64? - #os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] - os: ["ubuntu-latest", "windows-latest", "linux-arm64"] + os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] include: - os: ubuntu-latest os_name: linux target_arch: x64 exe_ext: "" - #- os: macos-latest - # os_name: osx - # target_arch: x64 - # exe_ext: "" + - os: macos-latest + os_name: osx + target_arch: x64 + exe_ext: "" - os: windows-latest os_name: win target_arch: x64 From 92df824af4defb9c66e0d8b252304e2fb601b754 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 12:52:13 -0700 Subject: [PATCH 134/181] Try simplified install for x265 Now that the Windows build environment is working and is much simpler than it was when we first got x265 working, try a simple "make install" on Windows again for x265. --- .github/workflows/release.yaml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5b09f20..6144d9c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -192,16 +192,7 @@ jobs: -DENABLE_SHARED=OFF \ -DENABLE_CLI=OFF make - # TODO: Re-evaluate this now that I have simplified the environment. - # For some reason, make install isn't working for this project on - # Windows. I don't know whether to x265's build system or the crazy - # Actions Windows build environment. Either way, these generic - # instructions work for all platforms. - $SUDO mkdir -p /usr/local/include - $SUDO mkdir -p /usr/local/lib/pkgconfig - $SUDO cp x265_config.h ../source/x265.h /usr/local/include/ - $SUDO cp libx265.a /usr/local/lib/ - $SUDO cp x265.pc /usr/local/lib/pkgconfig/ + $SUDO make install # This adjustment to the x265 linker flags is needed, at least on # arm, to successfully link against it statically. (-lgcc_s not # found (or needed), and -lpthread missing) From c0c95dd9c728add8ad114c0c0ee760bcee4d68f9 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 16:03:43 -0700 Subject: [PATCH 135/181] Install CA certs on Windows Trying to resolve an SSL error when uploading assets to the release. --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6144d9c..054f7ab 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -118,6 +118,7 @@ jobs: pacman -Sy --noconfirm \ git \ mercurial \ + mingw-w64-ca-certificates \ nasm \ yasm # Make sure that cmake builds makefiles and not ninja files. From d4bfc3c26fe7b799738037debc9a7c76cb7060a3 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 16:13:28 -0700 Subject: [PATCH 136/181] Correct CA cert package name on Windows --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 054f7ab..2df4029 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -118,7 +118,7 @@ jobs: pacman -Sy --noconfirm \ git \ mercurial \ - mingw-w64-ca-certificates \ + mingw-w64-x86_64-ca-certificates \ nasm \ yasm # Make sure that cmake builds makefiles and not ninja files. From 3952d5ac43f3ea889cf365f51e8d94a0e87a5ddc Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 17:07:48 -0700 Subject: [PATCH 137/181] Unlink homebrew packages on mac These pre-installed homebrew packages conflict with our static libraries. --- .github/workflows/release.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2df4029..137cfb4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -99,6 +99,13 @@ jobs: mercurial \ nasm \ yasm + # Unlink homebrew packages that conflict with our static library + # builds below. + brew unlink \ + lame \ + opus \ + opusfile \ + xz # Use sudo in install commands on macOS. echo "SUDO=sudo" >> "$GITHUB_ENV" From d9ec43313fdfa045be15a5584951240ebb519abb Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 18:48:13 -0700 Subject: [PATCH 138/181] Add debug step after asset upload Asset uploads are still failing on Windows, due to a lack of proper CA certs. --- .github/workflows/release.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 137cfb4..dc0ddb0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -364,6 +364,13 @@ jobs: token: ${{ secrets.SHAKA_BOT_TOKEN }} method: 'upload_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/")' + # TODO: remove once we fix CA certs on Windows + - name: Debug + uses: joeyparrish/action-tmate@v5 + with: + limit-access-to-actor: true + if: failure() + publish_release: name: Publish release needs: [draft_release, build] From eed9fdd21ae81c33bedd425b02883f505824731b Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 19:47:14 -0700 Subject: [PATCH 139/181] Fix dynamic linker checks on Mac Now everything on mac has been tested except uploading release assets --- .github/workflows/release.yaml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index dc0ddb0..ce071e5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -41,7 +41,6 @@ jobs: needs: draft_release strategy: matrix: - # TODO: Finish Mac # TODO: Add Mac arm64? # TODO: Add Windows arm64? os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] @@ -324,22 +323,17 @@ jobs: ldd ffmpeg.exe | grep -qvi /c/Windows/ && exit 1 ldd ffprobe.exe | grep -qvi /c/Windows/ && exit 1 elif [[ "${{ runner.os }}" == "macOS" ]]; then - # TODO: Verify the usage of otool for macOS. - otool -L ffmpeg && exit 1 - otool -L ffprobe && exit 1 + # These will still be dynamic executables, but they should not link + # against anything outside of /usr/lib or /System/Library. The + # grep command will succeed if it can find anything outside + # these two folders, and then we fail if that succeeds. + otool -L ffmpeg | grep '\t' | grep -Evq '(/System/Library|/usr/lib)' && exit 1 + otool -L ffprobe | grep '\t' | grep -Evq '(/System/Library|/usr/lib)' && exit 1 fi # After commands that we expect to fail, we still need a successful # command here to make this step a success. true - # TODO: send PR for arm64 and unfork - # TODO: remove once the workflow is working correctly - - name: Debug - uses: joeyparrish/action-tmate@v5 - with: - limit-access-to-actor: true - if: failure() - - name: Prepare assets run: | set -e @@ -364,6 +358,7 @@ jobs: token: ${{ secrets.SHAKA_BOT_TOKEN }} method: 'upload_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/")' + # TODO: unfork with arm64 PR lands # TODO: remove once we fix CA certs on Windows - name: Debug uses: joeyparrish/action-tmate@v5 From 5ad17673ba5376bfaec5067edf34e3d200bd4587 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 21:43:48 -0700 Subject: [PATCH 140/181] Debug python executable in custom action The custom action on Windows is still failing due to a certificate failure. But when I debug it via SSH, python does not seem to have this issue. So now I am wondering if we have multiple versions of python. --- .github/workflows/custom-github-repo-api-action/action.yaml | 1 + .github/workflows/release.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index 6a5715b..edd8cf1 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -127,4 +127,5 @@ runs: path = "/releases/{}".format(release_id) call_api(path, data, method="PATCH") + print("[DEBUG] python executable: ", sys.executable) # FIXME: Debug ${{ inputs.method }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ce071e5..08e3312 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -121,12 +121,14 @@ jobs: run: | set -e set -x + which -a python3 # FIXME: Debug pacman -Sy --noconfirm \ git \ mercurial \ mingw-w64-x86_64-ca-certificates \ nasm \ yasm + which -a python3 # FIXME: Debug # Make sure that cmake builds makefiles and not ninja files. echo "CMAKE_GENERATOR=MSYS Makefiles" >> "$GITHUB_ENV" # Make sure that pkg-config searches the expected path. From 61beb1d58a8e633bda650a29dbb4a731b217743f Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 10 Aug 2021 21:49:31 -0700 Subject: [PATCH 141/181] Add missing import to debug in python custom action --- .github/workflows/custom-github-repo-api-action/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml index edd8cf1..6bcdadd 100644 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ b/.github/workflows/custom-github-repo-api-action/action.yaml @@ -32,6 +32,7 @@ runs: import mimetypes import os import urllib.request + import sys def call_api(path, data=None, headers=None, method=None, get_json=True, upload=False): From ec9b1fca44f307ef58d57773b5ae8fc4dbf0fae9 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 06:34:44 -0700 Subject: [PATCH 142/181] Debug cert failures on Windows python3 I suspect there may be a difference between invoking python3 directly and doing it through bash. That would explain why it works for me when I debug via SSH, but it fails when the custom action runs. --- .github/workflows/release.yaml | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 08e3312..07e637f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -26,7 +26,6 @@ jobs: release_id: ${{ steps.draft_release.outputs.release_id }} steps: - name: Checkout repo - shell: bash run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - name: Draft release @@ -72,7 +71,6 @@ jobs: steps: - name: Install Linux packages if: runner.os == 'Linux' - shell: bash run: | set -e set -x @@ -89,7 +87,6 @@ jobs: - name: Install macOS packages if: runner.os == 'macOS' - shell: bash run: | set -e set -x @@ -121,19 +118,40 @@ jobs: run: | set -e set -x - which -a python3 # FIXME: Debug pacman -Sy --noconfirm \ git \ mercurial \ - mingw-w64-x86_64-ca-certificates \ nasm \ yasm - which -a python3 # FIXME: Debug # Make sure that cmake builds makefiles and not ninja files. echo "CMAKE_GENERATOR=MSYS Makefiles" >> "$GITHUB_ENV" # Make sure that pkg-config searches the expected path. echo "PKG_CONFIG_PATH=/usr/local/lib/pkgconfig" >> "$GITHUB_ENV" + - name: Debug python3 on Windows + shell: python3 {0} + run: | + import urllib.request + import sys + print("[DEBUG] Python executable", sys.executable) + try: + urllib.request.urlopen("https://api.github.com/repos/joeyparrish/static-ffmpeg-binaries/releases") + print("[DEBUG] urllib request OK") + except Exception as e: + print("[DEBUG] urllib request failed", e) + + - name: Debug python3 on Windows, part 2 + shell: bash -c python3 {0} + run: | + import urllib.request + import sys + print("[DEBUG] Python executable", sys.executable) + try: + urllib.request.urlopen("https://api.github.com/repos/joeyparrish/static-ffmpeg-binaries/releases") + print("[DEBUG] urllib request OK") + except Exception as e: + print("[DEBUG] urllib request failed", e) + - name: Install libvpx run: | set -e @@ -374,7 +392,6 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo - shell: bash run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src # First, we have to take the release out of draft mode. Then, we can @@ -395,7 +412,6 @@ jobs: - name: Calculate MD5 sums of all release assets id: md5sum - shell: bash run: | set -e set -x From 51d35caca9a8bd3d0bdafa538ccbb46e36719490 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 06:46:19 -0700 Subject: [PATCH 143/181] Rearrange one of the debugging steps Running "bash -c python3" as a shell did not do what I expected. It produced no output. Try an inline python script running from a bash shell, instead. If this works, we can shift the custom action into a standalone python script. --- .github/workflows/release.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 07e637f..8192bca 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -141,16 +141,16 @@ jobs: print("[DEBUG] urllib request failed", e) - name: Debug python3 on Windows, part 2 - shell: bash -c python3 {0} + shell: bash run: | - import urllib.request + python3 -c 'import urllib.request import sys print("[DEBUG] Python executable", sys.executable) try: urllib.request.urlopen("https://api.github.com/repos/joeyparrish/static-ffmpeg-binaries/releases") print("[DEBUG] urllib request OK") except Exception as e: - print("[DEBUG] urllib request failed", e) + print("[DEBUG] urllib request failed", e)' - name: Install libvpx run: | From cd79601a2256719222c601bf12489681b2748f62 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 08:39:15 -0700 Subject: [PATCH 144/181] Move shell default to the workflow level, rather than the job --- .github/workflows/release.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8192bca..d9a0bf5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -18,6 +18,10 @@ env: OPUS_VERSION: "1.3.1" VORBIS_VERSION: "1.3.7" +defaults: + run: + shell: bash + jobs: draft_release: name: Draft release @@ -64,10 +68,6 @@ jobs: name: Build ${{ matrix.os_name }} ${{ matrix.target_arch }} runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash - steps: - name: Install Linux packages if: runner.os == 'Linux' From a19242c7a4f91d2a0e596c76fc2021832e5c20cb Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 08:40:02 -0700 Subject: [PATCH 145/181] Try using GitHub API from node instead of Python If this works, we can trash the Python code and build something in node using the official GitHub client library. --- .github/workflows/release.yaml | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d9a0bf5..d26f8a3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -128,7 +128,7 @@ jobs: # Make sure that pkg-config searches the expected path. echo "PKG_CONFIG_PATH=/usr/local/lib/pkgconfig" >> "$GITHUB_ENV" - - name: Debug python3 on Windows + - name: Debug cert issues in python shell: python3 {0} run: | import urllib.request @@ -140,17 +140,22 @@ jobs: except Exception as e: print("[DEBUG] urllib request failed", e) - - name: Debug python3 on Windows, part 2 - shell: bash + - name: Debug cert issues in nodejs, part 1 run: | - python3 -c 'import urllib.request - import sys - print("[DEBUG] Python executable", sys.executable) - try: - urllib.request.urlopen("https://api.github.com/repos/joeyparrish/static-ffmpeg-binaries/releases") - print("[DEBUG] urllib request OK") - except Exception as e: - print("[DEBUG] urllib request failed", e)' + $SUDO npm install -g @octokit/rest + + - name: Debug cert issues in nodejs, part 2 + shell: nodejs {0} + run: | + const { Octokit } = require('@octokit/rest'); + const octokit = new Octokit(); + (async () => { + const { data } = await octokit.rest.repos.listReleases({ + owner: 'joeyparrish', + repo: 'static-ffmpeg-binaries', + }); + console.log(data); + })(); - name: Install libvpx run: | From 7492b09494c158d5d6b888e1ff16abd60fbb0e52 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 08:45:10 -0700 Subject: [PATCH 146/181] Attempt to correct nodejs shell name --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d26f8a3..f59cfa6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -145,7 +145,7 @@ jobs: $SUDO npm install -g @octokit/rest - name: Debug cert issues in nodejs, part 2 - shell: nodejs {0} + shell: node {0} run: | const { Octokit } = require('@octokit/rest'); const octokit = new Octokit(); From 2f21b3bedaaf5b353237213ae37a0f95c9846406 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 10:47:25 -0700 Subject: [PATCH 147/181] Try octokit/action instead of octokit/rest This follows docs for `@octokit/action` pretty closely. I hope it works this time. The last attempt did not, complaining that it could not find `@octokit/rest` at all, right after installing it. --- .github/workflows/release.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f59cfa6..d2f4759 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -142,19 +142,19 @@ jobs: - name: Debug cert issues in nodejs, part 1 run: | - $SUDO npm install -g @octokit/rest + npm install @octokit/action - name: Debug cert issues in nodejs, part 2 shell: node {0} run: | - const { Octokit } = require('@octokit/rest'); + const { Octokit } = require('@octokit/action'); const octokit = new Octokit(); (async () => { - const { data } = await octokit.rest.repos.listReleases({ + const response = await octokit.request("GET /repos/{owner}/{repo}/releases", { owner: 'joeyparrish', repo: 'static-ffmpeg-binaries', }); - console.log(data); + console.log(response); })(); - name: Install libvpx From 16cae39a4256e35cc3b3bce688752a5ea0aad6a5 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 11:05:41 -0700 Subject: [PATCH 148/181] Try using a separate node script The inline node script suffered from path-related issues and therefore could not import octokit. Having octokit installed at the same path as the script should alleviate this. --- .github/workflows/github-repo-api-client.js | 9 ++++++ .github/workflows/release.yaml | 35 +++++---------------- 2 files changed, 16 insertions(+), 28 deletions(-) create mode 100644 .github/workflows/github-repo-api-client.js diff --git a/.github/workflows/github-repo-api-client.js b/.github/workflows/github-repo-api-client.js new file mode 100644 index 0000000..dd80bf7 --- /dev/null +++ b/.github/workflows/github-repo-api-client.js @@ -0,0 +1,9 @@ +const { Octokit } = require('@octokit/action'); +const octokit = new Octokit(); +(async () => { + const response = await octokit.request("GET /repos/{owner}/{repo}/releases", { + owner: 'joeyparrish', + repo: 'static-ffmpeg-binaries', + }); + console.log(response); +})(); diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d2f4759..30feb55 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -128,34 +128,13 @@ jobs: # Make sure that pkg-config searches the expected path. echo "PKG_CONFIG_PATH=/usr/local/lib/pkgconfig" >> "$GITHUB_ENV" - - name: Debug cert issues in python - shell: python3 {0} - run: | - import urllib.request - import sys - print("[DEBUG] Python executable", sys.executable) - try: - urllib.request.urlopen("https://api.github.com/repos/joeyparrish/static-ffmpeg-binaries/releases") - print("[DEBUG] urllib request OK") - except Exception as e: - print("[DEBUG] urllib request failed", e) - - - name: Debug cert issues in nodejs, part 1 + - name: Debug cert issues in nodejs run: | + git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + cd repo-src/.github/workflows/ npm install @octokit/action - - - name: Debug cert issues in nodejs, part 2 - shell: node {0} - run: | - const { Octokit } = require('@octokit/action'); - const octokit = new Octokit(); - (async () => { - const response = await octokit.request("GET /repos/{owner}/{repo}/releases", { - owner: 'joeyparrish', - repo: 'static-ffmpeg-binaries', - }); - console.log(response); - })(); + node github-repo-api-client.js + false # FIXME: force failure - name: Install libvpx run: | @@ -373,8 +352,8 @@ jobs: wc -c * md5sum * - - name: Checkout repo - run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + #- name: Checkout repo + # run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - name: Attach assets to release uses: ./repo-src/.github/workflows/custom-github-repo-api-action From beff46ffc35da659ec23c4d7dbf408b1c0584428 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 11:10:21 -0700 Subject: [PATCH 149/181] Add npm, missing from self-hosted runner --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 30feb55..19309d6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -78,6 +78,7 @@ jobs: cmake \ mercurial \ nasm \ + npm \ pkg-config \ yasm \ libffmpeg-nvenc-dev \ From 7ab482cb0e86151c68e51ce625c754f18678c138 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 11:12:06 -0700 Subject: [PATCH 150/181] Remove node-based debugging octokit will work. Now I need to rewrite the API script. --- .github/workflows/release.yaml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 19309d6..d3b84c4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -129,14 +129,6 @@ jobs: # Make sure that pkg-config searches the expected path. echo "PKG_CONFIG_PATH=/usr/local/lib/pkgconfig" >> "$GITHUB_ENV" - - name: Debug cert issues in nodejs - run: | - git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - cd repo-src/.github/workflows/ - npm install @octokit/action - node github-repo-api-client.js - false # FIXME: force failure - - name: Install libvpx run: | set -e @@ -353,8 +345,8 @@ jobs: wc -c * md5sum * - #- name: Checkout repo - # run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + - name: Checkout repo + run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - name: Attach assets to release uses: ./repo-src/.github/workflows/custom-github-repo-api-action From 28eb45e0ca2ed3b406e33fd0b5ff4ec918b32b28 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 11:12:22 -0700 Subject: [PATCH 151/181] Unfork the tmate debugging action My PR for arm64 support landed! --- .github/workflows/release.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d3b84c4..9c520dd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -355,10 +355,9 @@ jobs: token: ${{ secrets.SHAKA_BOT_TOKEN }} method: 'upload_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/")' - # TODO: unfork with arm64 PR lands - # TODO: remove once we fix CA certs on Windows + # TODO: remove once we are done debugging - name: Debug - uses: joeyparrish/action-tmate@v5 + uses: mxschmitt/action-tmate@v3.6 with: limit-access-to-actor: true if: failure() From a79425cc63a139199288b314054f7ad3d71734b9 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 13:12:23 -0700 Subject: [PATCH 152/181] Rewrite the API client The custom action API client written in Python was not able to communicate with the GitHub servers from a Windows host due to CA cert errors. These went away when debugging via SSH. This completely rewrites the API client in node using octokit, which seems to work in all cases. --- .github/workflows/api-client/.gitignore | 1 + .github/workflows/api-client/main.js | 127 +++++++++++++++++ .../workflows/api-client/package-lock.json | 122 ++++++++++++++++ .github/workflows/api-client/package.json | 5 + .../custom-github-repo-api-action/action.yaml | 132 ------------------ .github/workflows/github-repo-api-client.js | 9 -- .github/workflows/release.yaml | 85 +++++------ 7 files changed, 292 insertions(+), 189 deletions(-) create mode 100644 .github/workflows/api-client/.gitignore create mode 100644 .github/workflows/api-client/main.js create mode 100644 .github/workflows/api-client/package-lock.json create mode 100644 .github/workflows/api-client/package.json delete mode 100644 .github/workflows/custom-github-repo-api-action/action.yaml delete mode 100644 .github/workflows/github-repo-api-client.js diff --git a/.github/workflows/api-client/.gitignore b/.github/workflows/api-client/.gitignore new file mode 100644 index 0000000..3c3629e --- /dev/null +++ b/.github/workflows/api-client/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/.github/workflows/api-client/main.js b/.github/workflows/api-client/main.js new file mode 100644 index 0000000..24a23d4 --- /dev/null +++ b/.github/workflows/api-client/main.js @@ -0,0 +1,127 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A script to communicate with the GitHub API to perform certain actions in +// the workflow. + +const fs = require('fs'); +const path = require('path'); +const { Octokit } = require('@octokit/core'); + +const repo = process.env['GITHUB_REPOSITORY']; + +const octokit = new Octokit({ + auth: process.env['GITHUB_TOKEN'], +}); + +const command = process.argv[2]; +const args = process.argv.slice(3); + + +const COMMAND_MAP = { + 'draft-release': draftRelease, + 'upload-all-assets': uploadAllAssets, + 'upload-asset': uploadAsset, + 'download-all-assets': downloadAllAssets, + 'publish-release': publishRelease, + 'update-release-body': updateReleaseBody, +}; + + +const method = COMMAND_MAP[command]; +if (!method) { + throw new Error(`Unknown command "${command}"`); +} + +(async () => { + const response = await method(...args); + // If there's a return value, print it. + if (response) { + console.log(response); + } +})(); + + +async function repoApiCall(method, apiPath, data) { + const url = `${method} /repos/${repo}${apiPath}`; + const response = await octokit.request(url, data); + return response.data; +} + +async function draftRelease(tagName) { + // Turns "refs/tags/foo" into "foo". + tagName = tagName.split('/').pop(); + + const response = await repoApiCall('POST', '/releases', { + tag_name: tagName, + name: tagName, + draft: true, + }); + + return response.id; +} + +async function uploadAsset(releaseId, assetPath) { + const baseName = path.basename(assetPath); + const data = await fs.promises.readFile(assetPath); + + const apiPath = `/releases/${releaseId}/assets?name=${baseName}`; + await repoApiCall('POST', apiPath, { + headers: { + 'content-type': 'application/octet-stream', + 'content-length': data.length, + }, + data, + }); +} + +async function uploadAllAssets(releaseId, folderPath) { + const folderContents = await fs.promises.readdir(folderPath); + for (const assetFilename of folderContents) { + const assetPath = path.join(folderPath, assetFilename); + await uploadAsset(releaseId, assetPath); + } +} + +async function downloadAllAssets(releaseId, outputPath) { + // If the output path does not exist, create it. + try { + await fs.promises.stat(outputPath); + } catch (error) { + await fs.promises.mkdir(outputPath); + } + + const apiPath = `/releases/${releaseId}/assets`; + const assetList = await repoApiCall('GET', apiPath); + for (const asset of assetList) { + const assetPath = path.join(outputPath, asset.name); + const outputStream = fs.createWriteSteam(assetPath); + + await new Promise((resolve, reject) => { + const request = https.request(asset.browser_download_url, (response) => { + response.pipe(outputStream); + }); + outputStream.on('finish', resolve); + request.on('error', reject); + }); + } +} + +async function publishRelease(releaseId) { + await repoApiCall('PATCH', `/releases/${releaseId}`, { draft: false }); +} + +async function updateReleaseBody(releaseId, body) { + await repoApiCall('PATCH', `/releases/${releaseId}`, { body }); +} diff --git a/.github/workflows/api-client/package-lock.json b/.github/workflows/api-client/package-lock.json new file mode 100644 index 0000000..4c919b8 --- /dev/null +++ b/.github/workflows/api-client/package-lock.json @@ -0,0 +1,122 @@ +{ + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "@octokit/auth-token": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.4.5.tgz", + "integrity": "sha512-BpGYsPgJt05M7/L/5FoE1PiAbdxXFZkX/3kDYcsvd1v6UhlnE5e96dTDr0ezX/EFwciQxf3cNV0loipsURU+WA==", + "requires": { + "@octokit/types": "^6.0.3" + } + }, + "@octokit/core": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.5.1.tgz", + "integrity": "sha512-omncwpLVxMP+GLpLPgeGJBF6IWJFjXDS5flY5VbppePYX9XehevbDykRH9PdCdvqt9TS5AOTiDide7h0qrkHjw==", + "requires": { + "@octokit/auth-token": "^2.4.4", + "@octokit/graphql": "^4.5.8", + "@octokit/request": "^5.6.0", + "@octokit/request-error": "^2.0.5", + "@octokit/types": "^6.0.3", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/endpoint": { + "version": "6.0.12", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.12.tgz", + "integrity": "sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA==", + "requires": { + "@octokit/types": "^6.0.3", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/graphql": { + "version": "4.6.4", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.6.4.tgz", + "integrity": "sha512-SWTdXsVheRmlotWNjKzPOb6Js6tjSqA2a8z9+glDJng0Aqjzti8MEWOtuT8ZSu6wHnci7LZNuarE87+WJBG4vg==", + "requires": { + "@octokit/request": "^5.6.0", + "@octokit/types": "^6.0.3", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/openapi-types": { + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-9.6.0.tgz", + "integrity": "sha512-L+8x7DpcNtHkMbTxxCxg3cozvHUNP46rOIzFwoMs0piWwQzAGNXqlIQO2GLvnKTWLUh99DkY+UyHVrP4jXlowg==" + }, + "@octokit/request": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.6.0.tgz", + "integrity": "sha512-4cPp/N+NqmaGQwbh3vUsYqokQIzt7VjsgTYVXiwpUP2pxd5YiZB2XuTedbb0SPtv9XS7nzAKjAuQxmY8/aZkiA==", + "requires": { + "@octokit/endpoint": "^6.0.1", + "@octokit/request-error": "^2.1.0", + "@octokit/types": "^6.16.1", + "is-plain-object": "^5.0.0", + "node-fetch": "^2.6.1", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/request-error": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz", + "integrity": "sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg==", + "requires": { + "@octokit/types": "^6.0.3", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "@octokit/types": { + "version": "6.25.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-6.25.0.tgz", + "integrity": "sha512-bNvyQKfngvAd/08COlYIN54nRgxskmejgywodizQNyiKoXmWRAjKup2/LYwm+T9V0gsKH6tuld1gM0PzmOiB4Q==", + "requires": { + "@octokit/openapi-types": "^9.5.0" + } + }, + "before-after-hook": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.2.tgz", + "integrity": "sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ==" + }, + "deprecation": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" + }, + "is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==" + }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "requires": { + "wrappy": "1" + } + }, + "universal-user-agent": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", + "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + } + } +} diff --git a/.github/workflows/api-client/package.json b/.github/workflows/api-client/package.json new file mode 100644 index 0000000..94e3904 --- /dev/null +++ b/.github/workflows/api-client/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "@octokit/core": "^3.5.1" + } +} diff --git a/.github/workflows/custom-github-repo-api-action/action.yaml b/.github/workflows/custom-github-repo-api-action/action.yaml deleted file mode 100644 index 6bcdadd..0000000 --- a/.github/workflows/custom-github-repo-api-action/action.yaml +++ /dev/null @@ -1,132 +0,0 @@ -name: GitHub Repo API Action - -description: | - A reusable action to talk to the GitHub API, wihtout relying on external - actions that could be affected by a supply-chain attack. - -inputs: - username: - description: A username for authentication to the GitHub API. - required: true - token: - description: A token for authentication to the GitHub API. - required: true - method: - description: A high-level method to invoke. See action source for details. - required: true - -outputs: - release_id: - description: The ID of a newly-created release from the draft_release method - value: ${{ steps.call-repo-api.outputs.release_id }} - -runs: - using: composite - steps: - - name: Call Repo API - id: call-repo-api - shell: python3 {0} - run: | - import base64 - import json - import mimetypes - import os - import urllib.request - import sys - - def call_api(path, data=None, headers=None, method=None, get_json=True, - upload=False): - repo = os.environ["GITHUB_REPOSITORY"] - if upload: - url = "https://uploads.github.com/repos/{}{}".format(repo, path) - else: - url = "https://api.github.com/repos/{}{}".format(repo, path) - login = "${{ inputs.username }}:${{ inputs.token }}" - - if not headers: - headers = {} - if not method: - method = "POST" if data else "GET" - - # For debugging: - #print({ - # "method": method, - # "path": path, - # "url": url, - # "headers": headers, - # "data": data, - #}) - - auth_token = base64.b64encode(login.encode("utf-8")).decode("utf-8") - headers["Authorization"] = "Basic {}".format(auth_token) - - if type(data) is dict: - data_bytes = json.dumps(data).encode("utf-8") - headers["Content-Type"] = "application/json" - elif type(data) is str: - data_bytes = data.encode("utf-8") - elif not data: - data_bytes = None - else: - assert(type(data) is bytes) - data_bytes = data - - req = urllib.request.Request(url, data_bytes, headers) - with urllib.request.urlopen(req) as resp: - if get_json: - body = json.load(resp) - return body - else: - return resp.read() - - def draft_release(tag_name): - # Turns "refs/tags/foo" into "foo" - tag_name = tag_name.split("/").pop() - - data = { - "tag_name": tag_name, - "name": tag_name, - "draft": True, - } - body = call_api("/releases", data) - #print("Draft release body:", body) - release_id = str(body["id"]) - print("::set-output name=release_id::{}".format(release_id)) - - def upload_asset(release_id, asset_path, mime_type): - base_name = os.path.basename(asset_path) - path = "/releases/{}/assets?name={}".format(release_id, base_name) - with open(asset_path, "rb") as f: - data = f.read() - headers = {"Content-Type": mime_type} - call_api(path, data, headers, upload=True) - - def upload_all_assets(release_id, folder): - for asset in os.listdir(folder): - asset_path = os.path.join(folder, asset) - # This returns a tuple of (type, encoding), where type can be None - mime_type = mimetypes.guess_type(asset_path)[0] - if not mime_type: - mime_type = "application/octet-stream" - upload_asset(release_id, asset_path, mime_type) - - def download_all_assets(release_id, output_path): - if not os.path.exists(output_path): - os.mkdir(output_path) - - path = "/releases/{}/assets".format(release_id) - asset_list = call_api(path) - for asset in asset_list: - name = asset["name"] - url = asset["browser_download_url"] - #print("Downloading: ", url) - with urllib.request.urlopen(url) as download: - with open(os.path.join(output_path, name), "wb") as local_file: - local_file.write(download.read()) - - def update_release(release_id, **data): - path = "/releases/{}".format(release_id) - call_api(path, data, method="PATCH") - - print("[DEBUG] python executable: ", sys.executable) # FIXME: Debug - ${{ inputs.method }} diff --git a/.github/workflows/github-repo-api-client.js b/.github/workflows/github-repo-api-client.js deleted file mode 100644 index dd80bf7..0000000 --- a/.github/workflows/github-repo-api-client.js +++ /dev/null @@ -1,9 +0,0 @@ -const { Octokit } = require('@octokit/action'); -const octokit = new Octokit(); -(async () => { - const response = await octokit.request("GET /repos/{owner}/{repo}/releases", { - owner: 'joeyparrish', - repo: 'static-ffmpeg-binaries', - }); - console.log(response); -})(); diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9c520dd..2d476de 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -29,16 +29,17 @@ jobs: outputs: release_id: ${{ steps.draft_release.outputs.release_id }} steps: - - name: Checkout repo - run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - - name: Draft release id: draft_release - uses: ./repo-src/.github/workflows/custom-github-repo-api-action - with: - username: shaka-bot - token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: 'draft_release("${{ github.ref }}")' + env: + GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} + run: | + set -e + set -x + git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + tag="${{ github.ref }}" + release_id=$(node ./repo-src/.github/workflows/api-client/main.js draft-release "$tag") + echo "::set-output name=release_id::$release_id" build: needs: draft_release @@ -345,15 +346,16 @@ jobs: wc -c * md5sum * - - name: Checkout repo - run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - - name: Attach assets to release - uses: ./repo-src/.github/workflows/custom-github-repo-api-action - with: - username: shaka-bot - token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: 'upload_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/")' + env: + GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} + run: | + set -e + set -x + git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + release_id="${{ needs.draft_release.outputs.release_id }}" + node ./repo-src/.github/workflows/api-client/main.js \ + upload-all-assets "$release_id" assets/ # TODO: remove once we are done debugging - name: Debug @@ -367,39 +369,26 @@ jobs: needs: [draft_release, build] runs-on: ubuntu-latest steps: - - name: Checkout repo - run: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - - # First, we have to take the release out of draft mode. Then, we can - # download the assets and use their MD5 sums to update the release notes. - name: Publish release - uses: ./repo-src/.github/workflows/custom-github-repo-api-action - with: - username: shaka-bot - token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: 'update_release("${{ needs.draft_release.outputs.release_id }}", draft=False)' - - - name: Get all release assets - uses: ./repo-src/.github/workflows/custom-github-repo-api-action - with: - username: shaka-bot - token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: 'download_all_assets("${{ needs.draft_release.outputs.release_id }}", "assets/")' - - - name: Calculate MD5 sums of all release assets - id: md5sum + env: + GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} run: | set -e set -x - cd assets - SUMS=$(md5sum *) - echo "::set-output name=sums::$SUMS" - # The sums are now in an output variable that we can use in the next - # step to set the release body. - - - name: Update release notes - uses: ./repo-src/.github/workflows/custom-github-repo-api-action - with: - username: shaka-bot - token: ${{ secrets.SHAKA_BOT_TOKEN }} - method: 'update_release("${{ needs.draft_release.outputs.release_id }}", body="MD5 sums:\n${{ steps.md5sum.outputs.sums }}")' + git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + + # First, we have to take the release out of draft mode. Until then, + # we can't get download URLs for the assets. + release_id="${{ needs.draft_release.outputs.release_id }}" + node ./repo-src/.github/workflows/api-client/main.js \ + publish-release "$release_id" + + # Next, download the assets and compute their MD5 sums. + node ./repo-src/.github/workflows/api-client/main.js \ + download-all-assets "$release_id" assets/ + sums=$(md5sum *) + + # Finally, update the release body with the MD5 sums. + body=$(echo -e "MD5 sums:\n$sums") + node ./repo-src/.github/workflows/api-client/main.js \ + update-release-body "$release_id" "$body" From d6171c9accd2a21d49ab8ae733eb8ca9a2526572 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 13:14:02 -0700 Subject: [PATCH 153/181] Install deps before running the new node client --- .github/workflows/release.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2d476de..c5aa5eb 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -37,6 +37,7 @@ jobs: set -e set -x git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + (cd repo-src/.github/workflows/api-client && npm install) tag="${{ github.ref }}" release_id=$(node ./repo-src/.github/workflows/api-client/main.js draft-release "$tag") echo "::set-output name=release_id::$release_id" @@ -353,6 +354,7 @@ jobs: set -e set -x git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + (cd repo-src/.github/workflows/api-client && npm install) release_id="${{ needs.draft_release.outputs.release_id }}" node ./repo-src/.github/workflows/api-client/main.js \ upload-all-assets "$release_id" assets/ @@ -376,6 +378,7 @@ jobs: set -e set -x git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + (cd repo-src/.github/workflows/api-client && npm install) # First, we have to take the release out of draft mode. Until then, # we can't get download URLs for the assets. From 83741d6c1429f0370f1dd3c93e64ad0cc01bcfb6 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 13:29:49 -0700 Subject: [PATCH 154/181] Add a license header, comments, and whitespace This should make the workflow more readable. --- .github/workflows/release.yaml | 116 +++++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c5aa5eb..864df51 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,3 +1,18 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A workflow to build and release fresh binaries. name: Release # Runs when a new tag is created. Creates a release for that tag, then builds @@ -8,6 +23,8 @@ on: tags: - "*" +# These are the versions of ffmpeg and its dependencies that we build from +# source. env: FFMPEG_TAG: "n4.4" LIBVPX_TAG: "v1.9.0" @@ -18,11 +35,18 @@ env: OPUS_VERSION: "1.3.1" VORBIS_VERSION: "1.3.7" +# By default, run all commands in a bash shell. On Windows, the default would +# otherwise be powershell. Each shell command should begin with "set -e" (to +# make any failed command fail the script immediately) and "set -x" (to log +# what commands are being run). defaults: run: shell: bash jobs: + # On a single Linux host, draft a release. Later, different hosts will build + # for each OS/CPU in parallel, and then attach the resulting binaries to this + # draft. draft_release: name: Draft release runs-on: ubuntu-latest @@ -36,19 +60,34 @@ jobs: run: | set -e set -x + + # Check out this repo and install node deps so that we can run the + # API client. git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src (cd repo-src/.github/workflows/api-client && npm install) + + # Create a draft release associated with the tag that triggered this + # workflow. tag="${{ github.ref }}" release_id=$(node ./repo-src/.github/workflows/api-client/main.js draft-release "$tag") echo "::set-output name=release_id::$release_id" + # On several different hosts, build ffmpeg's dependencies, then ffmpeg itself. + # The deps are all built as static libraries. build: needs: draft_release strategy: matrix: # TODO: Add Mac arm64? # TODO: Add Windows arm64? + # These are the OS images that we will run. linux-arm64 is a + # self-hosted runner, as mid-2021, GitHub still does not offer arm64 + # VMs. os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] + + # Associate additional properties with each of these OS options. + # Commenting out an OS above is not enough to remove one. Its section + # here must also be commented out. include: - os: ubuntu-latest os_name: linux @@ -76,6 +115,11 @@ jobs: run: | set -e set -x + + # Install missing packages on Linux. + # TODO: Some of these are already on GitHub's VMs, but not our + # self-hosted runner. Try to make the self-hosted runner image more + # compatible with what GitHub offers by default. sudo apt -y install \ cmake \ mercurial \ @@ -85,6 +129,7 @@ jobs: yasm \ libffmpeg-nvenc-dev \ libvdpau-dev + # Use sudo in install commands on Linux. echo "SUDO=sudo" >> "$GITHUB_ENV" @@ -93,18 +138,24 @@ jobs: run: | set -e set -x + + # Use homebrew to install missing packages on mac. brew install \ md5sha1sum \ mercurial \ nasm \ yasm + # Unlink homebrew packages that conflict with our static library - # builds below. + # builds below. They are still installed, but will no longer be + # symlinked into default library paths, and the ffmpeg build will not + # pick up shared libraries we don't want. brew unlink \ lame \ opus \ opusfile \ xz + # Use sudo in install commands on macOS. echo "SUDO=sudo" >> "$GITHUB_ENV" @@ -112,7 +163,8 @@ jobs: if: runner.os == 'Windows' run: | # At this point, we're running Git Bash. After this step, we will be - # running msys bash, just as we would be in tmate. + # running msys bash, just as we would be when debugging via SSH with + # mxschmitt/action-tmate. echo "C:\\msys64\\usr\\bin" >> "$GITHUB_PATH" echo "C:\\msys64\\mingw64\\bin" >> "$GITHUB_PATH" @@ -121,13 +173,17 @@ jobs: run: | set -e set -x + + # Install msys packages we will need. pacman -Sy --noconfirm \ git \ mercurial \ nasm \ yasm + # Make sure that cmake builds makefiles and not ninja files. echo "CMAKE_GENERATOR=MSYS Makefiles" >> "$GITHUB_ENV" + # Make sure that pkg-config searches the expected path. echo "PKG_CONFIG_PATH=/usr/local/lib/pkgconfig" >> "$GITHUB_ENV" @@ -135,8 +191,10 @@ jobs: run: | set -e set -x + git clone --depth 1 https://chromium.googlesource.com/webm/libvpx -b "$LIBVPX_TAG" cd libvpx + # NOTE: disabling unit tests and examples significantly reduces build # time (by 80% as tested on a Jetson Nano) ./configure \ @@ -147,6 +205,7 @@ jobs: --disable-examples \ --enable-static \ --disable-shared + make $SUDO make install @@ -154,9 +213,13 @@ jobs: run: | set -e set -x + git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" + + # AOM insists on being built out-of-tree. mkdir aom_build cd aom_build + # NOTE: without CMAKE_INSTALL_PREFIX on Windows, files are installed # to c:\Program Files. cmake ../aom \ @@ -168,8 +231,10 @@ jobs: -DENABLE_TOOLS=OFF \ -DCONFIG_RUNTIME_CPU_DETECT=1 \ -DCONFIG_SHARED=0 + make $SUDO make install + # This adjustment to the aom linker flags is needed, at least on # arm, to successfully link against it statically. (-lm missing) $SUDO sed -e 's/-laom/-laom -lm/' -i.bk /usr/local/lib/pkgconfig/aom.pc @@ -178,10 +243,14 @@ jobs: run: | set -e set -x + git clone --depth 1 https://code.videolan.org/videolan/x264.git -b "$X264_TAG" cd x264 + ./configure \ --enable-static + + # Only build and install the static library. make libx264.a $SUDO make install-lib-static @@ -189,16 +258,20 @@ jobs: run: | set -e set -x + hg clone http://hg.videolan.org/x265 -r "$X265_TAG" cd x265/build + # NOTE: without CMAKE_INSTALL_PREFIX on Windows, files are installed # to c:\Program Files. cmake ../source \ -DCMAKE_INSTALL_PREFIX=/usr/local \ -DENABLE_SHARED=OFF \ -DENABLE_CLI=OFF + make $SUDO make install + # This adjustment to the x265 linker flags is needed, at least on # arm, to successfully link against it statically. (-lgcc_s not # found (or needed), and -lpthread missing) @@ -208,9 +281,11 @@ jobs: run: | set -e set -x + curl -L -o lame-"$LAME_VERSION".tar.gz https://sourceforge.net/projects/lame/files/lame/"$LAME_VERSION"/lame-"$LAME_VERSION".tar.gz/download tar xzf lame-"$LAME_VERSION".tar.gz cd lame-"$LAME_VERSION" + # Only build and install the library (--disable-front-end). The # frontend doesn't build on Windows, and we don't need it anyway. # On Windows, somehow prefix defaults to / instead of /usr/local, but @@ -220,6 +295,7 @@ jobs: --disable-frontend \ --enable-static \ --disable-shared + make $SUDO make install @@ -227,13 +303,16 @@ jobs: run: | set -e set -x + curl -LO https://archive.mozilla.org/pub/opus/opus-"$OPUS_VERSION".tar.gz tar xzf opus-"$OPUS_VERSION".tar.gz cd opus-"$OPUS_VERSION" + # On Windows, we can't link later if we build with -D_FORTIFY_SOURCE # now. But there is no configure option for this, so we edit the # configure script instead. sed -e 's/-D_FORTIFY_SOURCE=2//' -i.bk configure + # On Windows, somehow prefix defaults to / instead of /usr/local, but # only on some projects. No idea why. # On Windows, we also need to disable-stack-protector. @@ -243,8 +322,10 @@ jobs: --disable-stack-protector \ --enable-static \ --disable-shared + make $SUDO make install + # The pkgconfig linker flags for static opus don't work when ffmpeg # checks for opus in configure. Linking libm after libopus fixes it. $SUDO sed -e 's/-lopus/-lopus -lm/' -i.bk /usr/local/lib/pkgconfig/opus.pc @@ -253,8 +334,11 @@ jobs: run: | set -e set -x + git clone --depth 1 https://git.ffmpeg.org/ffmpeg.git -b "$FFMPEG_TAG" cd ffmpeg + + # Set some OS-specific environment variables and flags. if [[ "${{ runner.os }}" == "Linux" ]]; then export CFLAGS="-static" export LDFLAGS="-static" @@ -288,6 +372,7 @@ jobs: # don't specify this. PLATFORM_CONFIGURE_FLAGS="--target-os=mingw64" fi + ./configure \ --pkg-config-flags="--static" \ --disable-ffplay \ @@ -302,13 +387,17 @@ jobs: --enable-version3 \ --enable-static \ $PLATFORM_CONFIGURE_FLAGS + make + # No "make install" for ffmpeg. - name: Check that executables are static run: | set -e set -x + cd ffmpeg + if [[ "${{ runner.os }}" == "Linux" ]]; then # If ldd succeeds, then these are dynamic executables, so we fail # this step if ldd succeeds. @@ -329,18 +418,23 @@ jobs: otool -L ffmpeg | grep '\t' | grep -Evq '(/System/Library|/usr/lib)' && exit 1 otool -L ffprobe | grep '\t' | grep -Evq '(/System/Library|/usr/lib)' && exit 1 fi - # After commands that we expect to fail, we still need a successful - # command here to make this step a success. + + # After commands that we expect to fail (greps and ldd commands + # above), we still need a successful command at the end of the script + # to make this step of the workflow a success. true - name: Prepare assets run: | set -e set -x + mkdir assets + SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}${{ matrix.exe_ext}}" cp ffmpeg/ffmpeg assets/ffmpeg"$SUFFIX" cp ffmpeg/ffprobe assets/ffprobe"$SUFFIX" + # Show sizes and MD5 sums that can be verified by users later if they # want to check for authenticity. cd assets @@ -353,8 +447,16 @@ jobs: run: | set -e set -x + + # Check out this repo and install node deps so that we can run the + # API client. git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src (cd repo-src/.github/workflows/api-client && npm install) + + # Attach the build outputs to the draft release. Each machine will + # do this separately and in parallel. Later, another job will take + # over to collect them all and use their MD5 sums to create the + # release notes (the "body" of the release). release_id="${{ needs.draft_release.outputs.release_id }}" node ./repo-src/.github/workflows/api-client/main.js \ upload-all-assets "$release_id" assets/ @@ -377,6 +479,9 @@ jobs: run: | set -e set -x + + # Check out this repo and install node deps so that we can run the + # API client. git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src (cd repo-src/.github/workflows/api-client && npm install) @@ -391,7 +496,8 @@ jobs: download-all-assets "$release_id" assets/ sums=$(md5sum *) - # Finally, update the release body with the MD5 sums. + # Finally, update the release notes (the "body" of the release) with + # the MD5 sums. body=$(echo -e "MD5 sums:\n$sums") node ./repo-src/.github/workflows/api-client/main.js \ update-release-body "$release_id" "$body" From 50434ea0a7c28aa7959f8e3f6c0cf0956e240701 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 13:59:35 -0700 Subject: [PATCH 155/181] Add comments and a self-documenting commands to api client Now the api client will show usage information if it is used incorrectly, and there are more comments in the source. --- .github/workflows/api-client/main.js | 107 +++++++++++++++++++++------ 1 file changed, 85 insertions(+), 22 deletions(-) diff --git a/.github/workflows/api-client/main.js b/.github/workflows/api-client/main.js index 24a23d4..b049a5e 100644 --- a/.github/workflows/api-client/main.js +++ b/.github/workflows/api-client/main.js @@ -17,6 +17,8 @@ const fs = require('fs'); const path = require('path'); + +// octokit is the official API client of GitHub. const { Octokit } = require('@octokit/core'); const repo = process.env['GITHUB_REPOSITORY']; @@ -25,40 +27,51 @@ const octokit = new Octokit({ auth: process.env['GITHUB_TOKEN'], }); -const command = process.argv[2]; -const args = process.argv.slice(3); - +const COMMAND_MAP = {}; -const COMMAND_MAP = { - 'draft-release': draftRelease, - 'upload-all-assets': uploadAllAssets, - 'upload-asset': uploadAsset, - 'download-all-assets': downloadAllAssets, - 'publish-release': publishRelease, - 'update-release-body': updateReleaseBody, -}; +// Convert a camelCase name to kebab-case. +function camelCaseToKebabCase(name) { + // Split the camelCase name into parts with a zero-length lookahead regex on + // any capital letter. Something like "methodName" should be split into + // ["method", "Name"]. + const nameParts = name.split(/(?=[A-Z])/); -const method = COMMAND_MAP[command]; -if (!method) { - throw new Error(`Unknown command "${command}"`); + // Convert those parts into a kebab-case name. + return nameParts.map(part => part.toLowerCase()).join('-'); } -(async () => { - const response = await method(...args); - // If there's a return value, print it. - if (response) { - console.log(response); - } -})(); - +// Register a method that the user can invoke on the command-line. We use +// (cheap) introspection to find the argument names, so that we can +// automatically document usage of each command without worrying about the docs +// getting out of sync with the code. +function registerCommand(method) { + const methodName = method.name; + const commandName = camelCaseToKebabCase(methodName); + + // Hack out the arguments from the stringified function. This is terrible + // and will not work in the general case of all JavaScript, but it does work + // here. (Don't be like me.) + const firstLine = method.toString().split('\n')[0]; + const argString = firstLine.split('(')[1].split(')')[0]; + const camelArgs = argString.replace(/\s+/, '').split(','); + const args = camelArgs.map(camelCaseToKebabCase); + + COMMAND_MAP[commandName] = { + commandName, + method, + args, + }; +} +// A helper function to make calls to the GitHub Repo API. async function repoApiCall(method, apiPath, data) { const url = `${method} /repos/${repo}${apiPath}`; const response = await octokit.request(url, data); return response.data; } + async function draftRelease(tagName) { // Turns "refs/tags/foo" into "foo". tagName = tagName.split('/').pop(); @@ -71,6 +84,7 @@ async function draftRelease(tagName) { return response.id; } +registerCommand(draftRelease); async function uploadAsset(releaseId, assetPath) { const baseName = path.basename(assetPath); @@ -85,6 +99,7 @@ async function uploadAsset(releaseId, assetPath) { data, }); } +// Not registered as an independent command. async function uploadAllAssets(releaseId, folderPath) { const folderContents = await fs.promises.readdir(folderPath); @@ -93,6 +108,7 @@ async function uploadAllAssets(releaseId, folderPath) { await uploadAsset(releaseId, assetPath); } } +registerCommand(uploadAllAssets); async function downloadAllAssets(releaseId, outputPath) { // If the output path does not exist, create it. @@ -117,11 +133,58 @@ async function downloadAllAssets(releaseId, outputPath) { }); } } +registerCommand(downloadAllAssets); async function publishRelease(releaseId) { await repoApiCall('PATCH', `/releases/${releaseId}`, { draft: false }); } +registerCommand(publishRelease); async function updateReleaseBody(releaseId, body) { await repoApiCall('PATCH', `/releases/${releaseId}`, { body }); } +registerCommand(updateReleaseBody); + + +// We expect a command and arguments. +const commandName = process.argv[2]; +const args = process.argv.slice(3); +const command = COMMAND_MAP[commandName]; +let okay = true; + +if (!commandName) { + console.error('No command selected!'); + okay = false; +} else if (!command) { + console.error(`Unknown command: ${commandName}`); + okay = false; +} else if (args.length != command.args.length) { + console.error(`Wrong number of arguments for command: ${commandName}`); + okay = false; +} + +// If there is no command name, there will also be no command, so this usage +// section applies to both conditions above. +if (!okay) { + console.error(''); + console.error('Usage:'); + const thisScript = path.basename(process.argv[1]); + + for (possibleCommand of Object.values(COMMAND_MAP)) { + console.error( + ' ', + thisScript, + possibleCommand.commandName, + ...possibleCommand.args.map(arg => `<${arg}>`)); + } + process.exit(1); +} + +// Run the command with the given arguments. +(async () => { + const response = await command.method(...args); + // If there's a return value, print it. + if (response) { + console.log(response); + } +})(); From e4a3c64d0bf87e58156ec03b8d22cf117a08b9fc Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 14:25:02 -0700 Subject: [PATCH 156/181] Add error handling to the api client In the most recent workflow run, there was an upload error, but it was not caught by the workflow and treated as an error. This fixes that so that the workflow would fail correctly. It does not, however, fix the actual upload failure. --- .github/workflows/api-client/main.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/api-client/main.js b/.github/workflows/api-client/main.js index b049a5e..05019ce 100644 --- a/.github/workflows/api-client/main.js +++ b/.github/workflows/api-client/main.js @@ -182,7 +182,17 @@ if (!okay) { // Run the command with the given arguments. (async () => { - const response = await command.method(...args); + let response; + + try { + response = await command.method(...args); + } catch (error) { + console.error('Command failed!'); + console.error(''); + console.error(error); + process.exit(1); + } + // If there's a return value, print it. if (response) { console.log(response); From 7dbd706593916f72bc2e44979602824862262808 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 14:25:33 -0700 Subject: [PATCH 157/181] Fix upload calls Asset uploads go to a different API endpoint. --- .github/workflows/api-client/main.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/api-client/main.js b/.github/workflows/api-client/main.js index 05019ce..e64f58b 100644 --- a/.github/workflows/api-client/main.js +++ b/.github/workflows/api-client/main.js @@ -65,9 +65,18 @@ function registerCommand(method) { } // A helper function to make calls to the GitHub Repo API. -async function repoApiCall(method, apiPath, data) { +async function repoApiCall(method, apiPath, data, upload=false) { const url = `${method} /repos/${repo}${apiPath}`; - const response = await octokit.request(url, data); + + // Clone the "data" passed in. + const options = Object.assign({}, data); + + // If we're uploading, that goes to a different API endpoint. + if (upload) { + options.baseUrl = 'https://uploads.github.com'; + } + + const response = await octokit.request(url, options); return response.data; } @@ -97,7 +106,7 @@ async function uploadAsset(releaseId, assetPath) { 'content-length': data.length, }, data, - }); + }, /* upload= */ true); } // Not registered as an independent command. From 8f61f896164d872234a4b7e24a9031ed42a7c912 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 14:28:06 -0700 Subject: [PATCH 158/181] Comment out the SSH debug step --- .github/workflows/release.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 864df51..e65f6a6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -461,12 +461,12 @@ jobs: node ./repo-src/.github/workflows/api-client/main.js \ upload-all-assets "$release_id" assets/ - # TODO: remove once we are done debugging - - name: Debug - uses: mxschmitt/action-tmate@v3.6 - with: - limit-access-to-actor: true - if: failure() + # NOTE: Uncomment this step to debug failures via SSH. + #- name: Debug + # uses: mxschmitt/action-tmate@v3.6 + # with: + # limit-access-to-actor: true + # if: failure() publish_release: name: Publish release From c22d1e197f50e3a704e7564b8e9c08509607af0e Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 16:32:02 -0700 Subject: [PATCH 159/181] Fix download of assets in new API client There was a missing module, a method name typo, and we needed to handle redirects. --- .github/workflows/api-client/main.js | 39 ++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/.github/workflows/api-client/main.js b/.github/workflows/api-client/main.js index e64f58b..7faa964 100644 --- a/.github/workflows/api-client/main.js +++ b/.github/workflows/api-client/main.js @@ -16,6 +16,7 @@ // the workflow. const fs = require('fs'); +const https = require('https'); const path = require('path'); // octokit is the official API client of GitHub. @@ -29,6 +30,8 @@ const octokit = new Octokit({ const COMMAND_MAP = {}; +const MAX_REDIRECTS = 3; + // Convert a camelCase name to kebab-case. function camelCaseToKebabCase(name) { @@ -119,6 +122,30 @@ async function uploadAllAssets(releaseId, folderPath) { } registerCommand(uploadAllAssets); +// A helper function that will fetch via HTTPS and follow redirects. +function fetchViaHttps(url, outputStream, redirectCount=0) { + if (redirectCount > MAX_REDIRECTS) { + throw new Error('Too many redirects!'); + } + + return new Promise((resolve, reject) => { + const request = https.get(url, (response) => { + if (response.statusCode == 301 || response.statusCode == 302) { + // Handle HTTP redirects. + const newUrl = response.headers.location; + + resolve(fetchViaHttps(newUrl, outputStream, redirectCount + 1)); + } else if (response.statusCode == 200) { + response.pipe(outputStream); + outputStream.on('finish', resolve); + } else { + reject(new Error(`Bad HTTP status code: ${response.statusCode}`)); + } + }); + request.on('error', reject); + }); +} + async function downloadAllAssets(releaseId, outputPath) { // If the output path does not exist, create it. try { @@ -130,16 +157,12 @@ async function downloadAllAssets(releaseId, outputPath) { const apiPath = `/releases/${releaseId}/assets`; const assetList = await repoApiCall('GET', apiPath); for (const asset of assetList) { + const url = asset.browser_download_url; const assetPath = path.join(outputPath, asset.name); - const outputStream = fs.createWriteSteam(assetPath); + const outputStream = fs.createWriteStream(assetPath); - await new Promise((resolve, reject) => { - const request = https.request(asset.browser_download_url, (response) => { - response.pipe(outputStream); - }); - outputStream.on('finish', resolve); - request.on('error', reject); - }); + console.log(`Fetching ${url} to ${assetPath}`); + await fetchViaHttps(url, outputStream); } } registerCommand(downloadAllAssets); From 9f76d2598309f3be0faae1c0f896c7092de1bf93 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 17:32:10 -0700 Subject: [PATCH 160/181] Create more detailed release notes Now we will include the date, the versions of each piece of software, and the MD5 sums. --- .github/workflows/release.yaml | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e65f6a6..389a4f2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -192,6 +192,7 @@ jobs: set -e set -x + echo "libvpx $LIBVPX_TAG" >> versions-built.txt git clone --depth 1 https://chromium.googlesource.com/webm/libvpx -b "$LIBVPX_TAG" cd libvpx @@ -214,6 +215,7 @@ jobs: set -e set -x + echo "aom $AOM_TAG" >> versions-built.txt git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" # AOM insists on being built out-of-tree. @@ -244,6 +246,7 @@ jobs: set -e set -x + echo "x264 $X264_TAG" >> versions-built.txt git clone --depth 1 https://code.videolan.org/videolan/x264.git -b "$X264_TAG" cd x264 @@ -259,6 +262,7 @@ jobs: set -e set -x + echo "x265 $X265_TAG" >> versions-built.txt hg clone http://hg.videolan.org/x265 -r "$X265_TAG" cd x265/build @@ -282,6 +286,7 @@ jobs: set -e set -x + echo "lame $LAME_VERSION" >> versions-built.txt curl -L -o lame-"$LAME_VERSION".tar.gz https://sourceforge.net/projects/lame/files/lame/"$LAME_VERSION"/lame-"$LAME_VERSION".tar.gz/download tar xzf lame-"$LAME_VERSION".tar.gz cd lame-"$LAME_VERSION" @@ -304,6 +309,7 @@ jobs: set -e set -x + echo "opus $OPUS_VERSION" >> versions-built.txt curl -LO https://archive.mozilla.org/pub/opus/opus-"$OPUS_VERSION".tar.gz tar xzf opus-"$OPUS_VERSION".tar.gz cd opus-"$OPUS_VERSION" @@ -335,6 +341,7 @@ jobs: set -e set -x + echo "ffmpeg $FFMPEG_TAG" >> versions-built.txt git clone --depth 1 https://git.ffmpeg.org/ffmpeg.git -b "$FFMPEG_TAG" cd ffmpeg @@ -430,10 +437,12 @@ jobs: set -x mkdir assets + echo "$GITHUB_REPOSITORY {{ $github.ref }}" >> versions-built.txt SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}${{ matrix.exe_ext}}" cp ffmpeg/ffmpeg assets/ffmpeg"$SUFFIX" cp ffmpeg/ffprobe assets/ffprobe"$SUFFIX" + cp versions-built.txt assets/ # Show sizes and MD5 sums that can be verified by users later if they # want to check for authenticity. @@ -491,13 +500,21 @@ jobs: node ./repo-src/.github/workflows/api-client/main.js \ publish-release "$release_id" - # Next, download the assets and compute their MD5 sums. + # Next, download the assets. node ./repo-src/.github/workflows/api-client/main.js \ download-all-assets "$release_id" assets/ - sums=$(md5sum *) # Finally, update the release notes (the "body" of the release) with - # the MD5 sums. - body=$(echo -e "MD5 sums:\n$sums") + # the versions of the software we built, plus MD5 sums of the + # resulting binaries. + mv assets/versions-built.txt . + > body.txt + echo "Date: $(date -u)" >> body.txt + echo "" >> body.txt + echo "Versions built:" >> body.txt + cat versions-built.txt >> body.txt + echo "" >> body.txt + echo "MD5 sums:" >> body.txt + (cd assets; md5sum *) >> body.txt node ./repo-src/.github/workflows/api-client/main.js \ - update-release-body "$release_id" "$body" + update-release-body "$release_id" "$(cat body.txt)" From b07b9d8e63e02457cb75bddbb13258544dede0f9 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 20:30:45 -0700 Subject: [PATCH 161/181] Debug GitHub Actions environment variables The scheme I introduced in the previous commit where I collect the various software versions into a text file and upload it as an asset failed. This failed because the text file uploaded from one runner would not be able to overwrite a file of the same name. In general, this should be a good thing. We probably do not want an accidental duplicate to replace something from another runner. It would be better for the workflow to fail. But this means we should have another way to collect the information. I would prefer something that would never get out of sync with the environment variables, so rather than reference each one in the final job, I will see if there is a way I can collect their values in a script. This will fail quickly after printing the info I am interested in. --- .github/workflows/release.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 389a4f2..05cadc7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -110,6 +110,11 @@ jobs: runs-on: ${{ matrix.os }} steps: + - name: Debug GitHub environment variables + run: | + echo "{{ toJSON(env) }}" + exit 1 + - name: Install Linux packages if: runner.os == 'Linux' run: | @@ -471,11 +476,11 @@ jobs: upload-all-assets "$release_id" assets/ # NOTE: Uncomment this step to debug failures via SSH. - #- name: Debug - # uses: mxschmitt/action-tmate@v3.6 - # with: - # limit-access-to-actor: true - # if: failure() + - name: Debug + uses: mxschmitt/action-tmate@v3.6 + with: + limit-access-to-actor: true + if: failure() publish_release: name: Publish release From 0bc40dc8959d7e29c6a7e86e49ff03c5479a9b62 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 20:33:21 -0700 Subject: [PATCH 162/181] Another try at debugging the environment values --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 05cadc7..f19dbc9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -112,7 +112,7 @@ jobs: steps: - name: Debug GitHub environment variables run: | - echo "{{ toJSON(env) }}" + echo "${{ toJSON(env) }}" exit 1 - name: Install Linux packages From 735a1c02def71a4bbadd86ac63d7a9427755cddc Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 20:44:56 -0700 Subject: [PATCH 163/181] Rewrite software version collection for release notes --- .github/workflows/release.yaml | 43 ++++++++++++++-------------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f19dbc9..1d3d1a3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -110,11 +110,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Debug GitHub environment variables - run: | - echo "${{ toJSON(env) }}" - exit 1 - - name: Install Linux packages if: runner.os == 'Linux' run: | @@ -197,7 +192,6 @@ jobs: set -e set -x - echo "libvpx $LIBVPX_TAG" >> versions-built.txt git clone --depth 1 https://chromium.googlesource.com/webm/libvpx -b "$LIBVPX_TAG" cd libvpx @@ -220,7 +214,6 @@ jobs: set -e set -x - echo "aom $AOM_TAG" >> versions-built.txt git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" # AOM insists on being built out-of-tree. @@ -251,7 +244,6 @@ jobs: set -e set -x - echo "x264 $X264_TAG" >> versions-built.txt git clone --depth 1 https://code.videolan.org/videolan/x264.git -b "$X264_TAG" cd x264 @@ -267,7 +259,6 @@ jobs: set -e set -x - echo "x265 $X265_TAG" >> versions-built.txt hg clone http://hg.videolan.org/x265 -r "$X265_TAG" cd x265/build @@ -291,7 +282,6 @@ jobs: set -e set -x - echo "lame $LAME_VERSION" >> versions-built.txt curl -L -o lame-"$LAME_VERSION".tar.gz https://sourceforge.net/projects/lame/files/lame/"$LAME_VERSION"/lame-"$LAME_VERSION".tar.gz/download tar xzf lame-"$LAME_VERSION".tar.gz cd lame-"$LAME_VERSION" @@ -314,7 +304,6 @@ jobs: set -e set -x - echo "opus $OPUS_VERSION" >> versions-built.txt curl -LO https://archive.mozilla.org/pub/opus/opus-"$OPUS_VERSION".tar.gz tar xzf opus-"$OPUS_VERSION".tar.gz cd opus-"$OPUS_VERSION" @@ -346,7 +335,6 @@ jobs: set -e set -x - echo "ffmpeg $FFMPEG_TAG" >> versions-built.txt git clone --depth 1 https://git.ffmpeg.org/ffmpeg.git -b "$FFMPEG_TAG" cd ffmpeg @@ -442,12 +430,9 @@ jobs: set -x mkdir assets - echo "$GITHUB_REPOSITORY {{ $github.ref }}" >> versions-built.txt - SUFFIX="-${{ matrix.os_name }}-${{ matrix.target_arch }}${{ matrix.exe_ext}}" cp ffmpeg/ffmpeg assets/ffmpeg"$SUFFIX" cp ffmpeg/ffprobe assets/ffprobe"$SUFFIX" - cp versions-built.txt assets/ # Show sizes and MD5 sums that can be verified by users later if they # want to check for authenticity. @@ -476,11 +461,11 @@ jobs: upload-all-assets "$release_id" assets/ # NOTE: Uncomment this step to debug failures via SSH. - - name: Debug - uses: mxschmitt/action-tmate@v3.6 - with: - limit-access-to-actor: true - if: failure() + #- name: Debug + # uses: mxschmitt/action-tmate@v3.6 + # with: + # limit-access-to-actor: true + # if: failure() publish_release: name: Publish release @@ -512,14 +497,22 @@ jobs: # Finally, update the release notes (the "body" of the release) with # the versions of the software we built, plus MD5 sums of the # resulting binaries. - mv assets/versions-built.txt . > body.txt - echo "Date: $(date -u)" >> body.txt + + echo "Date:" >> body.txt + echo " $(date -u)" >> body.txt echo "" >> body.txt - echo "Versions built:" >> body.txt - cat versions-built.txt >> body.txt + + echo "$GITHUB_REPOSITORY version:" >> body.txt + echo " {{ $github.ref }}" >> body.txt echo "" >> body.txt + + echo "Software versions:" >> body.txt + echo "${{ toJSON(env) }}" >> body.txt + echo "" >> body.txt + echo "MD5 sums:" >> body.txt - (cd assets; md5sum *) >> body.txt + (cd assets; md5sum * | sed -e 's/^/ /') >> body.txt + node ./repo-src/.github/workflows/api-client/main.js \ update-release-body "$release_id" "$(cat body.txt)" From 6648bab00acd7bf1fca9f3e6ba5c2661c85ead4a Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 11 Aug 2021 20:47:42 -0700 Subject: [PATCH 164/181] Fix typo in release note compilation script --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1d3d1a3..9545a95 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -504,7 +504,7 @@ jobs: echo "" >> body.txt echo "$GITHUB_REPOSITORY version:" >> body.txt - echo " {{ $github.ref }}" >> body.txt + echo " ${{ github.ref }}" >> body.txt echo "" >> body.txt echo "Software versions:" >> body.txt From 8acbee12289907799b5286b4bb14c0b12f13b3fe Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 12 Aug 2021 08:50:31 -0700 Subject: [PATCH 165/181] Refactor versioning The previous commit resulted in a successful, complete workflow... that also leaked a token associated with shaka-bot. Oops! (The token has now been burned.) The issue was in using "env" to collect software versions. So rather than use the environment variables to store this information, it is now located in a text file that makes it easy to build into our release notes later. When checking out code for dependencies, we use this same file (through a shell script) to extract the version number we want to check out. This also changes the order of publication. The previous workflow run generated an email that included no real information, because we published the release before adding any release notes. However, the only real dependency between publication and release notes is that we need to publish the release before we can get MD5 sums for the binary. So now we compile the release notes without MD5 sums, then publish, then add the MD5 sums after publication. Finally, this changes the token we use to the automatic GITHUB_TOKEN. I am not sure what the result will be, though. What user name will be associated with the release? Mine? Some GitHub bot? Will it have all the appropriate permissions? My hope is that this mitigates any possibility of a leaked, persistent token in the future. --- .github/workflows/get-version.sh | 22 +++++++ .github/workflows/release.yaml | 104 +++++++++++++++++-------------- .github/workflows/versions.txt | 8 +++ 3 files changed, 88 insertions(+), 46 deletions(-) create mode 100755 .github/workflows/get-version.sh create mode 100644 .github/workflows/versions.txt diff --git a/.github/workflows/get-version.sh b/.github/workflows/get-version.sh new file mode 100755 index 0000000..78bc6e1 --- /dev/null +++ b/.github/workflows/get-version.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Pull a tag or version number from versions.txt. + +dir=$(dirname "$0") +key="$1" + +cat "$dir"/versions.txt | grep "^$key:" | sed -e 's/.*: //' diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9545a95..ddaa2fb 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -23,17 +23,7 @@ on: tags: - "*" -# These are the versions of ffmpeg and its dependencies that we build from -# source. -env: - FFMPEG_TAG: "n4.4" - LIBVPX_TAG: "v1.9.0" - AOM_TAG: "v3.1.2" - X264_TAG: "stable" - X265_TAG: "stable" - LAME_VERSION: "3.100" - OPUS_VERSION: "1.3.1" - VORBIS_VERSION: "1.3.7" +# NOTE: The versions of the software we build are stored in versions.txt. # By default, run all commands in a bash shell. On Windows, the default would # otherwise be powershell. Each shell command should begin with "set -e" (to @@ -56,14 +46,14 @@ jobs: - name: Draft release id: draft_release env: - GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e set -x # Check out this repo and install node deps so that we can run the # API client. - git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src -b "$GITHUB_REF" (cd repo-src/.github/workflows/api-client && npm install) # Create a draft release associated with the tag that triggered this @@ -110,6 +100,16 @@ jobs: runs-on: ${{ matrix.os }} steps: + - name: Check out repo source + run: | + set -e + set -x + + # Check out this repo and install node deps so that we can run the + # API client. + git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src -b "$GITHUB_REF" + (cd repo-src/.github/workflows/api-client && npm install) + - name: Install Linux packages if: runner.os == 'Linux' run: | @@ -192,7 +192,8 @@ jobs: set -e set -x - git clone --depth 1 https://chromium.googlesource.com/webm/libvpx -b "$LIBVPX_TAG" + tag=$(repo-src/.github/workflows/get-version.sh libvpx) + git clone --depth 1 https://chromium.googlesource.com/webm/libvpx -b "$tag" cd libvpx # NOTE: disabling unit tests and examples significantly reduces build @@ -214,7 +215,8 @@ jobs: set -e set -x - git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$AOM_TAG" + tag=$(repo-src/.github/workflows/get-version.sh aom) + git clone --depth 1 https://aomedia.googlesource.com/aom/ -b "$tag" # AOM insists on being built out-of-tree. mkdir aom_build @@ -244,7 +246,8 @@ jobs: set -e set -x - git clone --depth 1 https://code.videolan.org/videolan/x264.git -b "$X264_TAG" + tag=$(repo-src/.github/workflows/get-version.sh x264) + git clone --depth 1 https://code.videolan.org/videolan/x264.git -b "$tag" cd x264 ./configure \ @@ -259,7 +262,8 @@ jobs: set -e set -x - hg clone http://hg.videolan.org/x265 -r "$X265_TAG" + tag=$(repo-src/.github/workflows/get-version.sh x265) + hg clone http://hg.videolan.org/x265 -r "$tag" cd x265/build # NOTE: without CMAKE_INSTALL_PREFIX on Windows, files are installed @@ -282,9 +286,10 @@ jobs: set -e set -x - curl -L -o lame-"$LAME_VERSION".tar.gz https://sourceforge.net/projects/lame/files/lame/"$LAME_VERSION"/lame-"$LAME_VERSION".tar.gz/download - tar xzf lame-"$LAME_VERSION".tar.gz - cd lame-"$LAME_VERSION" + version=$(repo-src/.github/workflows/get-version.sh lame) + curl -L -o lame-"$version".tar.gz https://sourceforge.net/projects/lame/files/lame/"$version"/lame-"$version".tar.gz/download + tar xzf lame-"$version".tar.gz + cd lame-"$version" # Only build and install the library (--disable-front-end). The # frontend doesn't build on Windows, and we don't need it anyway. @@ -304,9 +309,10 @@ jobs: set -e set -x - curl -LO https://archive.mozilla.org/pub/opus/opus-"$OPUS_VERSION".tar.gz - tar xzf opus-"$OPUS_VERSION".tar.gz - cd opus-"$OPUS_VERSION" + version=$(repo-src/.github/workflows/get-version.sh opus) + curl -LO https://archive.mozilla.org/pub/opus/opus-"$version".tar.gz + tar xzf opus-"$version".tar.gz + cd opus-"$version" # On Windows, we can't link later if we build with -D_FORTIFY_SOURCE # now. But there is no configure option for this, so we edit the @@ -335,7 +341,8 @@ jobs: set -e set -x - git clone --depth 1 https://git.ffmpeg.org/ffmpeg.git -b "$FFMPEG_TAG" + tag=$(repo-src/.github/workflows/get-version.sh ffmpeg) + git clone --depth 1 https://git.ffmpeg.org/ffmpeg.git -b "$tag" cd ffmpeg # Set some OS-specific environment variables and flags. @@ -442,16 +449,11 @@ jobs: - name: Attach assets to release env: - GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e set -x - # Check out this repo and install node deps so that we can run the - # API client. - git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src - (cd repo-src/.github/workflows/api-client && npm install) - # Attach the build outputs to the draft release. Each machine will # do this separately and in parallel. Later, another job will take # over to collect them all and use their MD5 sums to create the @@ -474,7 +476,7 @@ jobs: steps: - name: Publish release env: - GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -e set -x @@ -484,35 +486,45 @@ jobs: git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src (cd repo-src/.github/workflows/api-client && npm install) - # First, we have to take the release out of draft mode. Until then, - # we can't get download URLs for the assets. - release_id="${{ needs.draft_release.outputs.release_id }}" - node ./repo-src/.github/workflows/api-client/main.js \ - publish-release "$release_id" - - # Next, download the assets. - node ./repo-src/.github/workflows/api-client/main.js \ - download-all-assets "$release_id" assets/ - - # Finally, update the release notes (the "body" of the release) with - # the versions of the software we built, plus MD5 sums of the - # resulting binaries. + # Compile the release notes (the "body" of the release) with the date + # and the versions of the software we built. > body.txt echo "Date:" >> body.txt echo " $(date -u)" >> body.txt echo "" >> body.txt + # Strip "refs/tags/foo" down to just "foo". + repo_tag=$(echo "${{ github.ref }}" | sed -e 's@.*/@@') echo "$GITHUB_REPOSITORY version:" >> body.txt - echo " ${{ github.ref }}" >> body.txt + echo " $repo_tag" >> body.txt echo "" >> body.txt echo "Software versions:" >> body.txt - echo "${{ toJSON(env) }}" >> body.txt + cat repo-src/.github/workflows/versions.txt | \ + sed -e 's/^/ /' >> body.txt echo "" >> body.txt + # Update the release notes with this preliminary version. This is + # what gets emailed out when we publish the release below. + node ./repo-src/.github/workflows/api-client/main.js \ + update-release-body "$release_id" "$(cat body.txt)" + + # Now we have to take the release out of draft mode. Until we do, we + # can't get download URLs for the assets. + release_id="${{ needs.draft_release.outputs.release_id }}" + node ./repo-src/.github/workflows/api-client/main.js \ + publish-release "$release_id" + + # Next, download the assets. + node ./repo-src/.github/workflows/api-client/main.js \ + download-all-assets "$release_id" assets/ + + # Now add the MD5 sums to the release notes. echo "MD5 sums:" >> body.txt (cd assets; md5sum * | sed -e 's/^/ /') >> body.txt + # Now update the release notes one last time, with the MD5 sums + # appended. node ./repo-src/.github/workflows/api-client/main.js \ update-release-body "$release_id" "$(cat body.txt)" diff --git a/.github/workflows/versions.txt b/.github/workflows/versions.txt new file mode 100644 index 0000000..facc780 --- /dev/null +++ b/.github/workflows/versions.txt @@ -0,0 +1,8 @@ +ffmpeg: n4.4 +libvpx: v1.9.0 +aom: v3.1.2 +x264: stable +x265: stable +lame: 3.100 +opus: 1.3.1 +vorbis: 1.3.7 From 5d53147ff4e1bd08935bfb1f25317170afe36c83 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 12 Aug 2021 09:00:25 -0700 Subject: [PATCH 166/181] Fix repo clone "git clone -b" can only take a tag or branch name like "foo", but not "refs/tags/foo". --- .github/workflows/release.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ddaa2fb..f098449 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -53,7 +53,8 @@ jobs: # Check out this repo and install node deps so that we can run the # API client. - git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src -b "$GITHUB_REF" + repo_tag=$(echo "$GITHUB_REF" | sed -e 's@.*/@@') + git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src -b "$repo_tag" (cd repo-src/.github/workflows/api-client && npm install) # Create a draft release associated with the tag that triggered this @@ -107,7 +108,8 @@ jobs: # Check out this repo and install node deps so that we can run the # API client. - git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src -b "$GITHUB_REF" + repo_tag=$(echo "$GITHUB_REF" | sed -e 's@.*/@@') + git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src -b "$repo_tag" (cd repo-src/.github/workflows/api-client && npm install) - name: Install Linux packages @@ -483,7 +485,8 @@ jobs: # Check out this repo and install node deps so that we can run the # API client. - git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src + repo_tag=$(echo "$GITHUB_REF" | sed -e 's@.*/@@') + git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src -b "$repo_tag" (cd repo-src/.github/workflows/api-client && npm install) # Compile the release notes (the "body" of the release) with the date @@ -494,8 +497,6 @@ jobs: echo " $(date -u)" >> body.txt echo "" >> body.txt - # Strip "refs/tags/foo" down to just "foo". - repo_tag=$(echo "${{ github.ref }}" | sed -e 's@.*/@@') echo "$GITHUB_REPOSITORY version:" >> body.txt echo " $repo_tag" >> body.txt echo "" >> body.txt From 35ff6f65b3c41a8ee244d4e9eed289c3a8af771e Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 12 Aug 2021 09:02:25 -0700 Subject: [PATCH 167/181] Check out repo source after installing missing packages Since we need "npm install", we should ensure npm is installed first. This specifically affected our self-hosted linux-arm64 runner, which does not have npm pre-installed. --- .github/workflows/release.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f098449..f78af0a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -101,17 +101,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - - name: Check out repo source - run: | - set -e - set -x - - # Check out this repo and install node deps so that we can run the - # API client. - repo_tag=$(echo "$GITHUB_REF" | sed -e 's@.*/@@') - git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src -b "$repo_tag" - (cd repo-src/.github/workflows/api-client && npm install) - - name: Install Linux packages if: runner.os == 'Linux' run: | @@ -189,6 +178,17 @@ jobs: # Make sure that pkg-config searches the expected path. echo "PKG_CONFIG_PATH=/usr/local/lib/pkgconfig" >> "$GITHUB_ENV" + - name: Check out repo source + run: | + set -e + set -x + + # Check out this repo and install node deps so that we can run the + # API client. + repo_tag=$(echo "$GITHUB_REF" | sed -e 's@.*/@@') + git clone --depth 1 https://github.com/"$GITHUB_REPOSITORY" repo-src -b "$repo_tag" + (cd repo-src/.github/workflows/api-client && npm install) + - name: Install libvpx run: | set -e From 43d848ac08cf5b0cc32602fe6225436dab3472f3 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 12 Aug 2021 09:11:51 -0700 Subject: [PATCH 168/181] Update README This brings the README in line with the latest workflow sources and other scripts. --- README.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index cad3850..c7843cf 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,10 @@ To download binaries, visit the [releases page][releases]. # License -The GitHub Actions workflows in this repo are covered by the Apache license. -Please see the [workflow source][source], and see [the Apache license][apache] +The GitHub Actions workflows and other scripts in this repo are covered by the +Apache license. +Please see the [workflow source][workflow], [API client source][api-client], +[version script source][version-script], and see [the Apache license][apache] for license details. The resulting FFmpeg binaries are built using GPL libraries, and are therefore @@ -25,17 +27,20 @@ Each run of the GitHub Actions workflow logs the MD5 sums of the binaries, and it places the MD5 sums into the release notes. You can see how they were built, and you can verify that they haven't been tampered with. The sums in the workflow logs, release notes, and the binaries should all match. -You can read the details in the [workflow source][source]. +You can read the details in the [workflow source][workflow]. # Triggering a build -Update the version numbers as needed in the [workflow][source], then create a -tag on the new commit. Full builds will be triggered, and binaries will be -attached to a release on the new tag. +Update the version numbers as needed in the [version file][version-file], then +create a tag on the new commit. Full builds will be triggered, and binaries +will be attached to a release on the new tag. [releases]: https://github.com/joeyparrish/static-ffmpeg-binaries/releases -[source]: https://github.com/joeyparrish/static-ffmpeg-binaries/blob/main/.github/workflows/release.yaml +[workflow]: https://github.com/joeyparrish/static-ffmpeg-binaries/blob/main/.github/workflows/release.yaml +[api-client]: https://github.com/joeyparrish/static-ffmpeg-binaries/blob/main/.github/workflows/api-client/main.js +[version-script]: https://github.com/joeyparrish/static-ffmpeg-binaries/blob/main/.github/workflows/get-version.sh +[version-file]: https://github.com/joeyparrish/static-ffmpeg-binaries/blob/main/.github/workflows/versions.txt [apache]: https://github.com/joeyparrish/static-ffmpeg-binaries/blob/main/LICENSE [gpl]: https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.GPLv3 From 1a59551e1b9a56e005a21bc102c98e47a020de3f Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 12 Aug 2021 09:20:32 -0700 Subject: [PATCH 169/181] Adjust date format in release notes Now dates will be formatted as YYYY-MM-DD --- .github/workflows/release.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f78af0a..26fcf10 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -493,8 +493,9 @@ jobs: # and the versions of the software we built. > body.txt + # The format provided by "date -I" is "YYYY-MM-DD". echo "Date:" >> body.txt - echo " $(date -u)" >> body.txt + echo " $(date -I)" >> body.txt echo "" >> body.txt echo "$GITHUB_REPOSITORY version:" >> body.txt From 1a9f38a3aa36b84b8e335840044e77d9511a4025 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 12 Aug 2021 09:31:29 -0700 Subject: [PATCH 170/181] Add docs on tag names --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index c7843cf..62b8845 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,16 @@ create a tag on the new commit. Full builds will be triggered, and binaries will be attached to a release on the new tag. +# Tag names + +Tag names should follow the form of `$FFMPEG_VERSION-$WORKFLOW_RELEASE_NUMBER`. +For example, the first time we release a build based on FFmpeg n4.4, the tag +should be "n4.4-1". If we need to update the dependencies, or change the +configuration, or make any other changes to the workflow that don't change the +FFmpeg version, the next release would be "n4.4-2". When FFmpeg n4.5 is +released upstream, we could update to that and then tag "n4.5-1". + + [releases]: https://github.com/joeyparrish/static-ffmpeg-binaries/releases [workflow]: https://github.com/joeyparrish/static-ffmpeg-binaries/blob/main/.github/workflows/release.yaml [api-client]: https://github.com/joeyparrish/static-ffmpeg-binaries/blob/main/.github/workflows/api-client/main.js From 03e085c41534b905bb1e0310dd66d5d5d3af55a4 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 12 Aug 2021 10:49:10 -0700 Subject: [PATCH 171/181] Fix missing release id The latest workflow refactor resulted in a missing release ID because a variable was evaluated at the wrong time. It is a simple fix, though, to move one line up. --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 26fcf10..e15417c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -509,12 +509,12 @@ jobs: # Update the release notes with this preliminary version. This is # what gets emailed out when we publish the release below. + release_id="${{ needs.draft_release.outputs.release_id }}" node ./repo-src/.github/workflows/api-client/main.js \ update-release-body "$release_id" "$(cat body.txt)" # Now we have to take the release out of draft mode. Until we do, we # can't get download URLs for the assets. - release_id="${{ needs.draft_release.outputs.release_id }}" node ./repo-src/.github/workflows/api-client/main.js \ publish-release "$release_id" From 0ba5bfab161280d3d4fbb5c8810b4f5457bb30aa Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 12 Aug 2021 12:15:42 -0700 Subject: [PATCH 172/181] Improve release notes formatting The leading spaces mean nothing when formatted as markdown. So add spaces and dashes to create indented lists. --- .github/workflows/release.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e15417c..0876f48 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -495,16 +495,16 @@ jobs: # The format provided by "date -I" is "YYYY-MM-DD". echo "Date:" >> body.txt - echo " $(date -I)" >> body.txt + echo " - $(date -I)" >> body.txt echo "" >> body.txt echo "$GITHUB_REPOSITORY version:" >> body.txt - echo " $repo_tag" >> body.txt + echo " - $repo_tag" >> body.txt echo "" >> body.txt echo "Software versions:" >> body.txt cat repo-src/.github/workflows/versions.txt | \ - sed -e 's/^/ /' >> body.txt + sed -e 's/^/ - /' >> body.txt echo "" >> body.txt # Update the release notes with this preliminary version. This is @@ -524,7 +524,7 @@ jobs: # Now add the MD5 sums to the release notes. echo "MD5 sums:" >> body.txt - (cd assets; md5sum * | sed -e 's/^/ /') >> body.txt + (cd assets; md5sum * | sed -e 's/^/ - /') >> body.txt # Now update the release notes one last time, with the MD5 sums # appended. From cdb5917b7dca8f0cadcbe15d598129ae589dd20f Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 12 Aug 2021 12:23:38 -0700 Subject: [PATCH 173/181] Disable workflow While I am debugging a weird issue with draft releases creating extra tags, I will disable the workflow. I will experiment with the api-client manually until I have it working correctly, then I will re-enable the workflow. --- .github/workflows/release.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0876f48..0fd248e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -18,10 +18,11 @@ name: Release # Runs when a new tag is created. Creates a release for that tag, then builds # ffmpeg and ffprobe on all OS & CPU combinations, then attaches them to the # release. -on: - push: - tags: - - "*" +# FIXME: re-enable this workflow +#on: +# push: +# tags: +# - "*" # NOTE: The versions of the software we build are stored in versions.txt. From 4e0291c397a210cb5fbbc5665e22b0b5efa635cb Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 12 Aug 2021 13:04:34 -0700 Subject: [PATCH 174/181] Fix extra tags created by updating release notes If you update the release body without specifying tag_name, it gets reset, resulting in a new tag being created with an auto-generated name like "untagged-SHA1". So we need to fetch the existing name before we update the body, and we need to specify it in the update call, too. This is not mentioned in GitHub's docs, and may be a bug on their end. --- .github/workflows/api-client/main.js | 16 +++++++++++++++- .github/workflows/release.yaml | 9 ++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/api-client/main.js b/.github/workflows/api-client/main.js index 7faa964..33771bc 100644 --- a/.github/workflows/api-client/main.js +++ b/.github/workflows/api-client/main.js @@ -173,10 +173,24 @@ async function publishRelease(releaseId) { registerCommand(publishRelease); async function updateReleaseBody(releaseId, body) { - await repoApiCall('PATCH', `/releases/${releaseId}`, { body }); + // NOTE: If you update the release body without specifying tag_name, it gets + // reset, resulting in a new tag being created with an auto-generated name + // like "untagged-SHA1". So we need to fetch the existing name before we + // update the body, and we need to specify it here. This is not mentioned in + // GitHub's docs, and may be a bug on their end. + const release = await getRelease(releaseId); + await repoApiCall('PATCH', `/releases/${releaseId}`, { + body, + tag_name: release.tag_name, + }); } registerCommand(updateReleaseBody); +async function getRelease(releaseId) { + return await repoApiCall('GET', `/releases/${releaseId}`); +} +registerCommand(getRelease); + // We expect a command and arguments. const commandName = process.argv[2]; diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0fd248e..0876f48 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -18,11 +18,10 @@ name: Release # Runs when a new tag is created. Creates a release for that tag, then builds # ffmpeg and ffprobe on all OS & CPU combinations, then attaches them to the # release. -# FIXME: re-enable this workflow -#on: -# push: -# tags: -# - "*" +on: + push: + tags: + - "*" # NOTE: The versions of the software we build are stored in versions.txt. From 0378f014bfe710ceba1455a2fd6c53692add6313 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 12 Aug 2021 17:38:30 -0700 Subject: [PATCH 175/181] Correct an outdated command in api-client --- .github/workflows/api-client/main.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/api-client/main.js b/.github/workflows/api-client/main.js index 33771bc..89368ce 100644 --- a/.github/workflows/api-client/main.js +++ b/.github/workflows/api-client/main.js @@ -209,8 +209,10 @@ if (!commandName) { okay = false; } -// If there is no command name, there will also be no command, so this usage -// section applies to both conditions above. +// If something is wrong with the way the script was called, print usage +// information. The list of commands and their arguments are gleaned from +// COMMAND_MAP, which was populated by registerCommand() and introspection of +// the command functions. if (!okay) { console.error(''); console.error('Usage:'); From 22902bb1eaa76de6ff7efd71fbd53a3b58319abf Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 12 Aug 2021 17:45:11 -0700 Subject: [PATCH 176/181] Clarify some workflow comments --- .github/workflows/release.yaml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0876f48..5e606e1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -137,10 +137,11 @@ jobs: nasm \ yasm - # Unlink homebrew packages that conflict with our static library - # builds below. They are still installed, but will no longer be - # symlinked into default library paths, and the ffmpeg build will not - # pick up shared libraries we don't want. + # Unlink pre-installed homebrew packages that conflict with our + # static library builds below. They are still installed, but will no + # longer be symlinked into default library paths, and the ffmpeg + # build will not pick up pre-installed shared libraries we don't want. + # Only our static versions will be used. brew unlink \ lame \ opus \ @@ -172,10 +173,11 @@ jobs: nasm \ yasm - # Make sure that cmake builds makefiles and not ninja files. + # Make sure that cmake generates makefiles and not ninja files. echo "CMAKE_GENERATOR=MSYS Makefiles" >> "$GITHUB_ENV" - # Make sure that pkg-config searches the expected path. + # Make sure that pkg-config searches the path where we will install + # things. echo "PKG_CONFIG_PATH=/usr/local/lib/pkgconfig" >> "$GITHUB_ENV" - name: Check out repo source @@ -296,7 +298,8 @@ jobs: # Only build and install the library (--disable-front-end). The # frontend doesn't build on Windows, and we don't need it anyway. # On Windows, somehow prefix defaults to / instead of /usr/local, but - # only on some projects. No idea why. + # only on some projects. No idea why that is the default on Windows, + # but --prefix=/usr/local fixes it. ./configure \ --prefix=/usr/local \ --disable-frontend \ @@ -322,7 +325,8 @@ jobs: sed -e 's/-D_FORTIFY_SOURCE=2//' -i.bk configure # On Windows, somehow prefix defaults to / instead of /usr/local, but - # only on some projects. No idea why. + # only on some projects. No idea why that is the default on Windows, + # but --prefix=/usr/local fixes it. # On Windows, we also need to disable-stack-protector. ./configure \ --prefix=/usr/local \ @@ -376,9 +380,9 @@ jobs: export LDFLAGS="-static -L/usr/local/lib" # Convince ffmpeg that we want to build for mingw64 (native - # Windows), not msys. Since we're in an msys environment, ffmpeg - # reasonably assumes we're building for that environment if we - # don't specify this. + # Windows), not msys (which involves some posix emulation). Since + # we're in an msys environment, ffmpeg reasonably assumes we're + # building for that environment if we don't specify this. PLATFORM_CONFIGURE_FLAGS="--target-os=mingw64" fi From 49dff7b936028841e4e876dbf2deee707c3d1ae0 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Mon, 16 Aug 2021 14:30:37 -0700 Subject: [PATCH 177/181] Add a note about the avoidance of third-party actions --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 62b8845..7c40159 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,9 @@ and you can verify that they haven't been tampered with. The sums in the workflow logs, release notes, and the binaries should all match. You can read the details in the [workflow source][workflow]. +No third-party GitHub Actions have been used in this workflow, to protect +against supply-chain attacks. + # Triggering a build From cc281925d4a0acf6c990ac67ad2abfb62f0ff0b7 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 25 Aug 2021 21:44:52 -0700 Subject: [PATCH 178/181] Responding to first round of feedback --- .github/workflows/api-client/main.js | 2 +- .github/workflows/release.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/api-client/main.js b/.github/workflows/api-client/main.js index 89368ce..c63f44d 100644 --- a/.github/workflows/api-client/main.js +++ b/.github/workflows/api-client/main.js @@ -86,7 +86,7 @@ async function repoApiCall(method, apiPath, data, upload=false) { async function draftRelease(tagName) { // Turns "refs/tags/foo" into "foo". - tagName = tagName.split('/').pop(); + tagName = tagName.replace('refs/tags/', ''); const response = await repoApiCall('POST', '/releases', { tag_name: tagName, diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5e606e1..25d1c32 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -495,7 +495,6 @@ jobs: # Compile the release notes (the "body" of the release) with the date # and the versions of the software we built. - > body.txt # The format provided by "date -I" is "YYYY-MM-DD". echo "Date:" >> body.txt From b5549f5354aa17b1899c5a84e55a6ebc2c3439e1 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Wed, 25 Aug 2021 21:48:47 -0700 Subject: [PATCH 179/181] Update and upgrade before installing packages on Linux An ubuntu package was "not found" in the previous trial run --- .github/workflows/release.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 25d1c32..c102849 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -111,6 +111,8 @@ jobs: # TODO: Some of these are already on GitHub's VMs, but not our # self-hosted runner. Try to make the self-hosted runner image more # compatible with what GitHub offers by default. + sudo apt -y update + sudo apt -y upgrade sudo apt -y install \ cmake \ mercurial \ From 93e912d296faa57984e360fc92b06c093476f0ba Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 26 Aug 2021 13:36:11 -0700 Subject: [PATCH 180/181] Responding to second round of feedback Fixed regex bug and renamed self-hosted runner tag --- .github/workflows/api-client/main.js | 2 +- .github/workflows/release.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/api-client/main.js b/.github/workflows/api-client/main.js index c63f44d..93c15d5 100644 --- a/.github/workflows/api-client/main.js +++ b/.github/workflows/api-client/main.js @@ -57,7 +57,7 @@ function registerCommand(method) { // here. (Don't be like me.) const firstLine = method.toString().split('\n')[0]; const argString = firstLine.split('(')[1].split(')')[0]; - const camelArgs = argString.replace(/\s+/, '').split(','); + const camelArgs = argString.replace(/\s+/g, '').split(','); const args = camelArgs.map(camelCaseToKebabCase); COMMAND_MAP[commandName] = { diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c102849..36462a3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -71,10 +71,10 @@ jobs: matrix: # TODO: Add Mac arm64? # TODO: Add Windows arm64? - # These are the OS images that we will run. linux-arm64 is a - # self-hosted runner, as mid-2021, GitHub still does not offer arm64 + # These are the OS images that we will run. self-hosted-linux-arm64 is + # a self-hosted runner, as mid-2021, GitHub still does not offer arm64 # VMs. - os: ["ubuntu-latest", "macos-latest", "windows-latest", "linux-arm64"] + os: ["ubuntu-latest", "macos-latest", "windows-latest", "self-hosted-linux-arm64"] # Associate additional properties with each of these OS options. # Commenting out an OS above is not enough to remove one. Its section @@ -92,7 +92,7 @@ jobs: os_name: win target_arch: x64 exe_ext: ".exe" - - os: linux-arm64 + - os: self-hosted-linux-arm64 os_name: linux target_arch: arm64 exe_ext: "" From 6857a56b693ddf0e29869d7dd1c2c309ce55d521 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 26 Aug 2021 15:21:46 -0700 Subject: [PATCH 181/181] Add sleep between publish and download Reduces flaky responses (404) when downloading assets. --- .github/workflows/release.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 36462a3..772c358 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -523,6 +523,11 @@ jobs: node ./repo-src/.github/workflows/api-client/main.js \ publish-release "$release_id" + # The downloads are sometimes a bit flaky (responding with 404) if we + # don't put some delay between publication and download. This number + # is arbitrary, but experimentally, it seems to solve the issue. + sleep 30 + # Next, download the assets. node ./repo-src/.github/workflows/api-client/main.js \ download-all-assets "$release_id" assets/