Merge pull request #143 from jacobweinstock/update-hookos-down

Update HookOS downloading:

## Description

<!--- Please describe what this PR is going to change -->
New HookOS download script enables downloading tar.gz and .iso files. It also allows for downloading based on architecture, kernel version, or extention.

## Why is this needed

<!--- Link to issue you have raised -->

Fixes: #

## How Has This Been Tested?
<!--- Please describe in detail how you tested your changes. -->
<!--- Include details of your testing environment, and the tests you ran to -->
<!--- see how your change affects other areas of the code, etc. -->


## How are existing users impacted? What migration steps/scripts do we need?

<!--- Fixes a bug, unblocks installation, removes a component of the stack etc -->
<!--- Requires a DB migration script, etc. -->


## Checklist:

I have:

- [ ] updated the documentation and/or roadmap (if required)
- [ ] added unit or e2e tests
- [ ] provided instructions on how to upgrade

tinkerbell/smee/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.5.0
+version: 0.5.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

tinkerbell/smee/templates/service.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.deploy }}
 ---
 apiVersion: v1
 kind: Service
@@ -27,3 +28,4 @@ spec:
     protocol: UDP
   selector:
     app: {{ .Values.name }}
+{{- end }}

tinkerbell/stack/Chart.lock

@@ -4,12 +4,12 @@ dependencies:
   version: 0.3.0
 - name: smee
   repository: file://../smee
-  version: 0.5.0
+  version: 0.5.1
 - name: rufio
   repository: file://../rufio
   version: 0.4.0
 - name: hegel
   repository: file://../hegel
   version: 0.4.0
-digest: sha256:73e9f43db380aeadd559122946600d32957bb680a458365320c4dca790c31568
-generated: "2024-10-30T12:38:29.599438269-06:00"
+digest: sha256:cd45c3fbb6ed074365833472021dba91d7716a23b3ff23e22f17f03b2ae4cd3d
+generated: "2024-11-22T21:01:58.580657087-07:00"

tinkerbell/stack/Chart.yaml

@@ -29,7 +29,7 @@ dependencies:
     version: "0.3.0"
     repository: "file://../tink"
   - name: smee
-    version: "0.5.0"
+    version: "0.5.1"
     repository: "file://../smee"
   - name: rufio
     version: "0.4.0"

tinkerbell/stack/templates/hook.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.stack.hook.enabled }}
+{{- if and .Values.stack.enabled .Values.stack.hook.enabled }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -8,64 +8,247 @@ metadata:
 data:
   entrypoint.sh: |-
     #!/usr/bin/env bash
-    # This script is designed to download the Hook artifacts.
-    set -x
-
-    function run_checksum_check() {
-      local checksum_file=$1
-
-      # releases of HookOS > 0.8.1 provided more binaries than we general want here.
-      # We only need the hook_aarch64 and hook_x86_64 binaries so we filter out the rest.
-      base_l="$(dirname ${checksum_file})"
-      (cd "${base_l}" && sed -i '/hook_x86_64\|hook_aarch64/!d' checksum.txt && sha512sum -c checksum.txt)
-      if [ $? -ne 0 ]; then
-        return 1
+    
+    set -euo pipefail
+    
+    function usage() {
+        echo "Usage: $0 [OPTION]..."
+        echo "Script for downloading HookOS artifacts"
+        echo
+        echo "Options:"
+        echo "  -u, --url                 Base URL to the location of the HookOS artifacts (default: https://github.com/tinkerbell/hook/releases/download/latest)"
+        echo "  -a, --arch                Architectures to download, one of [x86_64, aarch64, both] (default: both)"
+        echo "  -v, --version             The kernel version of the HookOS artifacts to download, one of [5.10, 6.6, both] (default: 6.6)"
+        echo "  -e, --ext                 The artifact extension types to download, one of [tar.gz, iso, both] (default: both)"
+        echo "  -s, --specific-artifacts  List of non-standard artifacts to download"
+        echo "  -o, --output-dir          Output directory to store the downloaded artifacts (default: .)"
+        echo "  -h, --help                Display this help and exit"
+    }
+    
+    function validate_option() {
+      local option="$1"
+      local valid_values="$2"
+      local value="$3"
+    
+      if [[ ! " ${valid_values[@]} " =~ " ${value} " ]]; then
+        >&2 echo "Invalid value: '$value' for '${option}'. Valid values are: [${valid_values[*]}]"
+        usage
+        exit 1
       fi
-      return 0
     }
-
+    
+    
+    function get_by_kernel_version() {
+        # data must be newline separated list of artifacts
+        local data="$1"
+        local version="$2"
+        local artifacts=""
+        
+        if [[ "${version}" == "both" ]]; then
+            artifacts+=$(grep -v "latest-lts" <<< $data)$'\n'
+            artifacts+=$(grep "latest-lts" <<< "$data")
+        elif [[ "${version}" == "6.6" ]]; then
+            artifacts=$(grep "latest-lts" <<< "$data")
+        elif [[ "${version}" == "5.10" ]]; then
+            artifacts=$(grep -v "latest-lts" <<< "$data")
+        fi
+        
+        echo "${artifacts}"
+    }
+    
+    function get_by_extension() {
+        local data="$1"
+        local ext="$2"
+        local artifacts=""
+        
+        if [[ "${ext}" == "both" ]]; then
+            artifacts=$(grep ".tar.gz" <<< "$data")$'\n'
+            artifacts+=$(grep ".iso" <<< "$data")
+        elif [[ "${ext}" == "tar.gz" ]]; then
+            artifacts=$(grep ".tar.gz" <<< "$data")
+        elif [[ "${ext}" == "iso" ]]; then
+            artifacts=$(grep ".iso" <<< "$data")
+        fi
+        
+        echo "${artifacts}"
+    }
+    
+    function get_by_arch() {
+        local data="$1"
+        local arch="$2"
+        local artifacts=""
+        
+        if [[ "${arch}" == "both" ]]; then
+            artifacts+=$(grep "x86_64" <<< "$data")$'\n'
+            artifacts+=$(grep "aarch64" <<< "$data")
+        elif [[ "${arch}" == "x86_64" ]]; then
+            artifacts=$(grep "x86_64" <<< "$data")
+        elif [[ "${arch}" == "aarch64" ]]; then
+            artifacts=$(grep "aarch64" <<< "$data")
+        fi
+        
+        echo "${artifacts}"
+    }
+    
+    function download_artifacts() {
+        local url="$1"
+        local artifacts="$2"
+        local out_dir="$3"
+        
+        while IFS= read -r line; do
+            wget -O "${out_dir}/${line}" "${url}/${line}"
+        done < <(printf '%s\n' "$artifacts")
+    }
+    
+    function run_checksum512() {
+        local checksum_data="$1"
+        local out_dir="$2"
+        
+        (cd "${out_dir}" && sha512sum -c <<< "${checksum_data}")
+        if [ $? -ne 0 ]; then
+          return 1
+        fi
+        return 0
+    }
+    
+    function checksum_format() {
+        # data is a newline separated list of artifacts
+        local data="$1"
+        local raw="$2"
+        local checksums=""
+    
+        while IFS= read -r line; do
+            checksums+=$(grep "${line}" <<< "${raw}")$'\n'
+        done < <(printf '%s\n' "$data")
+    
+        echo "${checksums}"
+    }
+    
+    # default values
+    url="https://github.com/tinkerbell/hook/releases/download/latest"
+    arch="both"
+    version="6.6"
+    ext="both"
+    specific_artifacts=()
+    output_dir="."
+    
+    # valid options
+    valid_arches=("x86_64" "aarch64" "both")
+    valid_versions=("5.10" "6.6" "both")
+    valid_exts=("tar.gz" "iso" "both")
+    
+    args=$(getopt -a -o u:a:v:e:s:o:h --long url:,arch:,version:,ext:,specific-artifacts:,output-dir:,help -- "$@")
+    if [[ $? -gt 0 ]]; then
+      usage
+    fi
+    
+    eval set -- ${args}
+    while :
+    do
+      case $1 in
+        -u | --url)
+          if [[ ! -z $2 ]]; then
+            url=$2
+          fi
+          shift 2 ;;
+        -a | --arch)
+          if [[ ! -z $2 ]]; then
+            validate_option "arch" "${valid_arches[*]}" $2
+            arch=$2
+          fi
+          shift 2 ;;
+        -v | --version)
+          if [[ ! -z $2 ]]; then
+            validate_option "version" "${valid_versions[*]}" $2
+            version=$2
+          fi
+          shift 2 ;;
+        -e | --ext)
+          if [[ ! -z $2 ]]; then
+            validate_option "ext" "${valid_exts[*]}" $2
+            ext=$2
+          fi
+          shift 2 ;;
+        -s | --specific-artifacts)
+          if [[ ! -z $2 ]]; then
+            specific_artifacts=$2
+          fi
+          shift 2 ;;
+        -o | --output-dir)
+          if [[ ! -z $2 ]]; then
+            output_dir=$2
+          fi
+          shift 2 ;;
+        -h | --help)
+          usage
+          exit 1
+          shift ;;
+        # -- means the end of the arguments; drop this, and break out of the while loop
+        --) shift; break ;;
+        *) >&2 echo Unsupported option: $1
+           usage ;;
+      esac
+    done
+    
+    echo "==> Downloading HookOS artifacts from ${url} for architecture(s): ${arch} and extension(s): ${ext} and version(s): ${version}"
+    
+    # 1. Download the checksum file
+    # 2. Generate a list of artifacts to download based on the options provided
+    # 3. Run a checksum check for all artifacts to be downloaded. For artifacts that pass (meaning the artifact is already downloaded), do nothing. Make a list of artifacts that need downloaded.
+    # 4. If all artifacts are already downloaded, sleep and wait for signals.
+    # 5. Download artifacts that need downloaded.
+    # 6. Run a checksum check for all downloaded artifacts.
+    # 7. If all checksums pass, sleep and wait for signals. If any checksum fails, exit with 1.
+    
     function main() {
-        # 1. Download the checksum.txt file
-        # 2. Check the checksums for files in the checksum.txt file
-        # 3. If the checksums match, sleep and wait for signals
-        # 4. If the checksums do not match, download the files again
-        # 5. Check the checksums again
-        local base_loc="$1"
-        local output_dir="$2"
-
-        # 1. Download the checksum.txt file
-        wget -O "${output_dir}/checksum.txt" "${base_loc}/checksum.txt"
-        # 2. Check the checksums for files in the checksum.txt file
-        run_checksum_check "${output_dir}/checksum.txt"
-        if [[ $? -eq 0 ]]; then
-        # 3. If the checksums match, sleep and wait for signals
-            sleep infinity & PID=$!
-            trap "kill $PID" INT TERM
-            wait $PID
-            return 0
+        local checksum_file="${output_dir}/checksum.txt"
+    
+        # 1.
+        echo "==> Downloading checksum file from ${url}"
+        if ! wget -O "${checksum_file}" ${url}/checksum.txt; then
+            echo "==> Failed to download checksum file: ${url}/checksum.txt"
+            return 1
         fi
-
-        # 4. If the checksums do not match, download the files again
-        files="$base_loc/hook_aarch64.tar.gz $base_loc/hook_x86_64.tar.gz"
-        for f in ${files}; do
-          echo "${f}"
-          wget -P "${output_dir}" "${f}"
-        done
-
-        run_checksum_check "${output_dir}/checksum.txt"
-        if [[ $? -eq 0 ]]; then
-        # 3. If the checksums match, sleep and wait for signals
+    
+        # 2.
+        echo "==> Parsing checksum file"
+        raw_data=$(cat "${checksum_file}")
+        data=$(cat "${checksum_file}" | awk '{print $2}')
+        by_kernel=$(get_by_kernel_version "${data}" "${version}")
+        by_extension=$(get_by_extension "${by_kernel}" "${ext}")
+        filtered=$(get_by_arch "${by_extension}" "${arch}")
+    
+        # 3.
+        echo "==> Running initial checksum check for all artifacts"
+        checksums=$(checksum_format "${filtered}" "${raw_data}")
+        if run_checksum512 "${checksums}" "${output_dir}"; then
             cd "${output_dir}"
-            for f in ${output_dir}/*.tar.gz; do tar --no-same-permissions --overwrite -ozxvf "${f}"; done
-            sleep infinity & PID=$!
-            trap "kill $PID" INT TERM
-            wait $PID
+            echo "==> Extracting existing artifacts"
+            for f in $(ls *.tar.gz | grep -vE "^dtbs"); do echo "==> Extracting ${f}"; tar --no-same-permissions --overwrite -ozxvf "${f}"; done
             return 0
         fi
-
-        echo "Checksums do not match. Exiting..."
-        return 1
+    
+        # 5.
+        echo "==> Downloading artifacts"
+        download_artifacts "${url}" "${filtered}" "${output_dir}"
+        checksums=$(checksum_format "${filtered}" "${raw_data}")
+        echo "==> Running checksum check for all downloaded artifacts"
+        if ! run_checksum512 "${checksums}" "${output_dir}"; then
+            echo "==> Checksum failed for some artifacts"
+            return 1
+        fi
+        cd "${output_dir}"
+        echo "==> Extracting artifacts"
+        for f in $(ls *.tar.gz | grep -vE "^dtbs"); do echo "==> Extracting ${f}"; tar --no-same-permissions --overwrite -ozxvf "${f}"; done
     }
-
-    main "{{ .Values.stack.hook.downloadURL }}" "/output"
+    
+    if ! main; then
+        exit 1
+    fi
+    
+    echo "==> All artifacts available, waiting for signals..."
+    sleep infinity & PID=$!
+    trap "kill $PID" INT TERM
+    wait $PID
+    
 {{- end }}

tinkerbell/stack/templates/nginx-configmap.yaml

@@ -1,6 +1,6 @@
+{{ if .Values.stack.enabled -}}
 # The NGINX ConfigMap is in a separate file because its checksum is used to trigger updates in
 # the deployment.
-{{ if .Values.stack.enabled -}}
 apiVersion: v1
 kind: ConfigMap
 metadata:

tinkerbell/stack/templates/nginx.yaml

@@ -118,6 +118,7 @@ spec:
       - name: download-hook
         image: {{ .Values.stack.hook.image }}
         command: ["/script/entrypoint.sh"]
+        args: ["--url", "{{ .Values.stack.hook.downloadURL }}", "--output-dir", "/output", "--arch", "{{ .Values.stack.hook.arch }}", "--version", "{{ .Values.stack.hook.kernelVersion }}", "--ext", "{{ .Values.stack.hook.extension }}"]
         volumeMounts:
           - mountPath: /output
             name: hook-artifacts

tinkerbell/stack/templates/nginx_pvc.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.stack.hook.enabled (empty .Values.stack.hook.persistence.existingClaim ) }}
+{{- if and .Values.stack.enabled .Values.stack.hook.enabled (empty .Values.stack.hook.persistence.existingClaim ) }}
 ---
 kind: StorageClass
 apiVersion: storage.k8s.io/v1

tinkerbell/stack/values.yaml

@@ -23,10 +23,13 @@ stack:
     enabled: true
     name: hook-files
     port: 8080
-    image: bash
+    image: bash:5.2.37
     # downloadURL only works with the > 0.8.1 Hook release because
     # previous Hook versions didn't provide a checksum file.
-    downloadURL: https://github.com/tinkerbell/hook/releases/download/v0.9.1
+    downloadURL: https://github.com/tinkerbell/hook/releases/download/v0.9.2
+    arch: both # x86_64, aarch64, both
+    extension: tar.gz # iso, tar.gz, both
+    kernelVersion: both # 5.10, 6.6, both
     persistence:
       # If existingClaim is set, the local persistence volume (localPersistentVolume) objects will NOT be created.
       # Use this to point to an existing production grade storage class.
@@ -39,7 +42,7 @@ stack:
         name: hook-artifacts
         accessModes:
           - ReadWriteMany
-        size: 1Gi
+        size: 2Gi
         extraLabels: {}
 
   kubevip: