.github/workflows: add beaglebone build and switch rpi to forrest runner

I've not updated qemux86 yet due to the pending key handing changes.

Signed-off-by: Jan Luebbe <[email protected]>

.github/workflows/generate.py

@@ -3,46 +3,62 @@
 from jinja2 import Template
 
 TEMPLATE = """
-name: «« layer »» CI
+name: build «« layer »»
 
 on:
-  # Trigger the workflow on push or pull request,
-  # but only for the master branch
   push:
     branches:
       - master
     paths:
+      - '.github/workflows/«« layer »».yml'
       - '«« layer »»/**'
   pull_request:
     branches:
       - master
     paths:
+      - '.github/workflows/«« layer »».yml'
       - '«« layer »»/**'
+  # allow rebuilding without a push
+  workflow_dispatch: {}
+
 jobs:
   build:
     name: «« layer »» Build
-    runs-on: ubuntu-20.04
+    # run on self-hosted runner for the main repo or if vars.BUILD_RUNS_ON is set
+    runs-on: >-
+      ${{
+        (vars.BUILD_RUNS_ON != ''  && fromJSON(vars.BUILD_RUNS_ON)) ||
+        (github.repository == 'rauc/meta-rauc-community' && fromJSON('["self-hosted", "forrest", "build"]')) ||
+        'ubuntu-20.04'
+      }}
     steps:
       - name: Install required packages
         run: |
-          sudo apt-get install diffstat
+          sudo apt-get -q -y --no-install-recommends install diffstat tree
       - name: Checkout
-        uses: actions/checkout@v3
-      «% for name, url in base_layers.items()|list + extra_layers.items()|list %»
-      - name: Clone «« name »»
-        run: git clone -b master «« url »»
+        uses: actions/checkout@v4
+      «% for layer_name, layer_info in layers.items() %»
+      - name: Clone «« layer_name »»
+        run: git clone --shared --reference-if-able /srv/shared-git/«« layer_name »».git -b «« layer_info["branch"] | default(release) »» «« layer_info["repo"] »»
       «% endfor %»
       - name: Initialize build directory
         run: |
           source poky/oe-init-build-env build
-          bitbake-layers add-layer ../meta-rauc
-          «% for name in extra_layers.keys() %»
-          bitbake-layers add-layer ../«« name »»
+          «% for layer_path in add_layers %»
+          bitbake-layers add-layer ../«« layer_path »»
           «% endfor %»
           bitbake-layers add-layer ../«« layer »»
-          echo 'INHERIT += "rm_work"' >> conf/local.conf
+          if [ -f ~/.yocto/auto.conf ]; then
+            cp ~/.yocto/auto.conf conf/
+            echo 'SOURCE_MIRROR_URL = "http://10.0.2.2/rauc-community/downloads"' >> conf/auto.conf
+          else
+            echo 'SSTATE_MIRRORS = "file://.* https://github-runner.pengutronix.de/rauc-community/sstate-cache/PATH"' >> conf/auto.conf
+            echo 'BB_SIGNATURE_HANDLER = "OEBasicHash"' >> conf/auto.conf
+            echo 'BB_HASHSERVE = ""' >> conf/auto.conf
+            echo 'OPKGBUILDCMD = "opkg-build -Z gzip -a -1n"' >> conf/auto.conf
+            echo 'INHERIT += "rm_work"' >> conf/auto.conf
+          fi
           echo 'DISTRO_FEATURES:remove = "alsa bluetooth usbgadget usbhost wifi nfs zeroconf pci 3g nfc x11 opengl ptest wayland vulkan"' >> conf/local.conf
-          echo 'SSTATE_MIRRORS = "file://.* http://195.201.147.117/sstate-cache/PATH"' >> conf/local.conf
           «% if machine %»
           echo 'MACHINE = "«« machine »»"' >> conf/local.conf
           «% endif %»
@@ -53,60 +69,152 @@
           «% for line in conf %»
           echo '«« line »»' >> conf/local.conf
           «% endfor %»
-      - name: bitbake parsing test
+      - name: Show configuration files
+        run: |
+          cd build/conf
+          rgrep . *.conf
+      - name: Test bitbake parsing
         run: |
           source poky/oe-init-build-env build
           bitbake -p
+      - name: Build rauc, rauc-native
+        run: |
+          source poky/oe-init-build-env build
+          bitbake rauc rauc-native
+      - name: Build «« image »»
+        run: |
+          source poky/oe-init-build-env build
+          bitbake «« image »»
+      - name: Build RAUC Bundle
+        run: |
+          source poky/oe-init-build-env build
+          bitbake «« bundle »»
+      - name: Cache Data
+        env:
+          CACHE_KEY: ${{ secrets.YOCTO_CACHE_KEY }}
+        if: ${{ env.CACHE_KEY }}
+        run: |
+          mkdir -p ~/.ssh
+          echo "$CACHE_KEY" >> ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          rsync -rvx --ignore-existing build/downloads rauc-community-cache: || true
+          rsync -rvx --ignore-existing build/sstate-cache rauc-community-cache: || true
+      - name: Show Artifacts
+        run: |
+          source poky/oe-init-build-env build
+          tree --du -h tmp/deploy/images || true
+      «% if artifacts %»
+      - name: Upload Artifacts
+        uses: jluebbe/forrest-upload-artifact@summary
+        with:
+          path: |
+            «% for artifact in artifacts %»
+            build/tmp/deploy/images/«« machine »»/«« artifact »»
+            «% endfor %»
+      «% endif %»
 """.lstrip()
 
 template = Template(
-        TEMPLATE,
-        block_start_string='«%',
-        block_end_string='%»',
-        variable_start_string='««',
-        variable_end_string='»»',
-        comment_start_string='«#',
-        comment_end_string='#»',
-        trim_blocks=True,
-        lstrip_blocks=True,
-        keep_trailing_newline=True,
-        )
+    TEMPLATE,
+    block_start_string="«%",
+    block_end_string="%»",
+    variable_start_string="««",
+    variable_end_string="»»",
+    comment_start_string="«#",
+    comment_end_string="#»",
+    trim_blocks=True,
+    lstrip_blocks=True,
+    keep_trailing_newline=True,
+)
 
-default_context = {
-    "base_layers": {
-        "poky": "git://git.yoctoproject.org/poky",
-        "meta-rauc": "https://github.com/rauc/meta-rauc.git"
+default_layers = {
+    "poky": {
+        "repo": "https://github.com/yoctoproject/poky.git",
+        # added by default
+        "add": [],
+    },
+    "meta-rauc": {
+        "repo": "https://github.com/rauc/meta-rauc.git",
     },
-    "extra_layers": {
+}
+
+default_context = {
+    "release": "master",
+    "layers": {
+        **default_layers,
     },
+    "extra_layers": {},
+    "add_layers": [],
     "machine": None,
     "conf": [],
+    "image": "core-image-minimal",
+    "bundle": "update-bundle",
+    "artifacts": []
 }
 
 contexts = [
     {
+        "layer": "meta-rauc-beaglebone",
         **default_context,
-        "layer": "meta-rauc-qemux86",
-        "fstypes": "tar.bz2 wic",
-        "wks_file": "qemux86-grub-efi.wks",
+        "release": "scarthgap",
+        "machine": "beaglebone-yocto",
+        "fstypes": "ext4 wic.zst",
+        "wks_file": "beaglebone-yocto-dual.wks.in",
         "conf": [
-            'EXTRA_IMAGEDEPENDS += "ovmf"',
-            'PREFERRED_RPROVIDER_virtual-grub-bootconf = "rauc-qemu-grubconf"',
+            'IMAGE_BOOT_FILES:append = " boot.scr"',
+        ],
+        "artifacts": [
+            "core-image-minimal-beaglebone-yocto.rootfs.wic.xz",
+            "core-image-minimal-beaglebone-yocto.rootfs.spdx.tar.zst",
+            "update-bundle-beaglebone-yocto.raucb",
         ],
     },
+    #{
+    #    "layer": "meta-rauc-qemux86",
+    #    **default_context,
+    #    "fstypes": "tar.bz2 wic.zst",
+    #    "wks_file": "qemux86-grub-efi.wks",
+    #    "conf": [
+    #        'EXTRA_IMAGEDEPENDS += "ovmf"',
+    #        'PREFERRED_RPROVIDER_virtual-grub-bootconf = "rauc-qemu-grubconf"',
+    #    ],
+    #    "bundle": "qemu-demo-bundle",
+    #    "artifacts": [
+    #        "core-image-minimal-qemux86-64.rootfs.wic.zst",
+    #    ],
+    #},
     {
-        **default_context,
         "layer": "meta-rauc-raspberrypi",
-        "extra_layers": {
-            "meta-raspberrypi": "git://git.yoctoproject.org/meta-raspberrypi",
+        **default_context,
+        "layers": {
+            **default_layers,
+            "meta-raspberrypi": {
+                "repo": "https://github.com/agherzan/meta-raspberrypi.git",
+            },
         },
         "machine": "raspberrypi4",
-        "fstypes": "ext4",
+        "fstypes": "ext4 wic.zst",
         "wks_file": "sdimage-dual-raspberrypi.wks.in",
+        "artifacts": [
+            "core-image-minimal-raspberrypi4.rootfs.ext4",
+            "core-image-minimal-raspberrypi4.rootfs.manifest",
+            "core-image-minimal-raspberrypi4.rootfs.spdx.json",
+            "core-image-minimal-raspberrypi4.rootfs.testdata.json",
+            "update-bundle-raspberrypi4.raucb",
+        ],
     },
 ]
 
 for context in contexts:
+    add_layers = context["add_layers"] = []
+    for k, v in context["layers"].items():
+        sub_layers = v.get("add")
+        if sub_layers is None:
+            add_layers.append(f"{k}")
+        else:
+            for add in sub_layers:
+                add_layers.append(f"{k}/{add}")
+
     output = template.render(context)
     file_name = f"{context['layer']}.yml"
     with open(file_name, "w") as file:

.github/workflows/meta-rauc-raspberrypi.yml

@@ -1,49 +1,105 @@
-name: meta-rauc-raspberrypi CI
+name: build meta-rauc-raspberrypi
 
 on:
-  # Trigger the workflow on push or pull request,
-  # but only for the master branch
   push:
     branches:
       - master
     paths:
+      - '.github/workflows/meta-rauc-raspberrypi.yml'
       - 'meta-rauc-raspberrypi/**'
   pull_request:
     branches:
       - master
     paths:
+      - '.github/workflows/meta-rauc-raspberrypi.yml'
       - 'meta-rauc-raspberrypi/**'
+  # allow rebuilding without a push
+  workflow_dispatch: {}
+
 jobs:
   build:
     name: meta-rauc-raspberrypi Build
-    runs-on: ubuntu-20.04
+    # run on self-hosted runner for the main repo or if vars.BUILD_RUNS_ON is set
+    runs-on: >-
+      ${{
+        (vars.BUILD_RUNS_ON != ''  && fromJSON(vars.BUILD_RUNS_ON)) ||
+        (github.repository == 'rauc/meta-rauc-community' && fromJSON('["self-hosted", "forrest", "build"]')) ||
+        'ubuntu-20.04'
+      }}
     steps:
       - name: Install required packages
         run: |
-          sudo apt-get install diffstat
+          sudo apt-get -q -y --no-install-recommends install diffstat tree
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Clone poky
-        run: git clone -b master git://git.yoctoproject.org/poky
+        run: git clone --shared --reference-if-able /srv/shared-git/poky.git -b master https://github.com/yoctoproject/poky.git
       - name: Clone meta-rauc
-        run: git clone -b master https://github.com/rauc/meta-rauc.git
+        run: git clone --shared --reference-if-able /srv/shared-git/meta-rauc.git -b master https://github.com/rauc/meta-rauc.git
       - name: Clone meta-raspberrypi
-        run: git clone -b master git://git.yoctoproject.org/meta-raspberrypi
+        run: git clone --shared --reference-if-able /srv/shared-git/meta-raspberrypi.git -b master https://github.com/agherzan/meta-raspberrypi.git
       - name: Initialize build directory
         run: |
           source poky/oe-init-build-env build
           bitbake-layers add-layer ../meta-rauc
           bitbake-layers add-layer ../meta-raspberrypi
           bitbake-layers add-layer ../meta-rauc-raspberrypi
-          echo 'INHERIT += "rm_work"' >> conf/local.conf
+          if [ -f ~/.yocto/auto.conf ]; then
+            cp ~/.yocto/auto.conf conf/
+            echo 'SOURCE_MIRROR_URL = "http://10.0.2.2/rauc-community/downloads"' >> conf/auto.conf
+          else
+            echo 'SSTATE_MIRRORS = "file://.* https://github-runner.pengutronix.de/rauc-community/sstate-cache/PATH"' >> conf/auto.conf
+            echo 'BB_SIGNATURE_HANDLER = "OEBasicHash"' >> conf/auto.conf
+            echo 'BB_HASHSERVE = ""' >> conf/auto.conf
+            echo 'OPKGBUILDCMD = "opkg-build -Z gzip -a -1n"' >> conf/auto.conf
+            echo 'INHERIT += "rm_work"' >> conf/auto.conf
+          fi
           echo 'DISTRO_FEATURES:remove = "alsa bluetooth usbgadget usbhost wifi nfs zeroconf pci 3g nfc x11 opengl ptest wayland vulkan"' >> conf/local.conf
-          echo 'SSTATE_MIRRORS = "file://.* http://195.201.147.117/sstate-cache/PATH"' >> conf/local.conf
           echo 'MACHINE = "raspberrypi4"' >> conf/local.conf
           echo 'DISTRO_FEATURES:append = " rauc"' >> conf/local.conf
           echo 'IMAGE_INSTALL:append = " rauc"' >> conf/local.conf
-          echo 'IMAGE_FSTYPES = "ext4"' >> conf/local.conf
+          echo 'IMAGE_FSTYPES = "ext4 wic.zst"' >> conf/local.conf
           echo 'WKS_FILE = "sdimage-dual-raspberrypi.wks.in"' >> conf/local.conf
-      - name: bitbake parsing test
+      - name: Show configuration files
+        run: |
+          cd build/conf
+          rgrep . *.conf
+      - name: Test bitbake parsing
         run: |
           source poky/oe-init-build-env build
           bitbake -p
+      - name: Build rauc, rauc-native
+        run: |
+          source poky/oe-init-build-env build
+          bitbake rauc rauc-native
+      - name: Build core-image-minimal
+        run: |
+          source poky/oe-init-build-env build
+          bitbake core-image-minimal
+      - name: Build RAUC Bundle
+        run: |
+          source poky/oe-init-build-env build
+          bitbake update-bundle
+      - name: Cache Data
+        env:
+          CACHE_KEY: ${{ secrets.YOCTO_CACHE_KEY }}
+        if: ${{ env.CACHE_KEY }}
+        run: |
+          mkdir -p ~/.ssh
+          echo "$CACHE_KEY" >> ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          rsync -rvx --ignore-existing build/downloads rauc-community-cache: || true
+          rsync -rvx --ignore-existing build/sstate-cache rauc-community-cache: || true
+      - name: Show Artifacts
+        run: |
+          source poky/oe-init-build-env build
+          tree --du -h tmp/deploy/images || true
+      - name: Upload Artifacts
+        uses: jluebbe/forrest-upload-artifact@summary
+        with:
+          path: |
+            build/tmp/deploy/images/raspberrypi4/core-image-minimal-raspberrypi4.rootfs.ext4
+            build/tmp/deploy/images/raspberrypi4/core-image-minimal-raspberrypi4.rootfs.manifest
+            build/tmp/deploy/images/raspberrypi4/core-image-minimal-raspberrypi4.rootfs.spdx.json
+            build/tmp/deploy/images/raspberrypi4/core-image-minimal-raspberrypi4.rootfs.testdata.json
+            build/tmp/deploy/images/raspberrypi4/update-bundle-raspberrypi4.raucb