Improve speed and reliability of repo management

.github/workflows/infrastructure-download-external.yml

@@ -65,6 +65,7 @@ env:
   GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   PR_NUMBER: ${{ github.event.number }}
   PACKAGES_URL: "https://fi.mirror.armbian.de/apt"
+  UPLOAD_PATH: "storage/incoming/external"
 
 jobs:
 
@@ -219,7 +220,7 @@ jobs:
           # Note: StrictHostKeychecking=no is used here; consider using proper known_hosts in production
           ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }} \
             ${{ inputs.HOST_USER }}@${{ inputs.HOST_DEPLOY }} \
-            "rm -rf storage/artifacts/*"
+            "rm -rf ${UPLOAD_PATH}/artifacts/*"
 
       - name: Checkout Armbian OS repository
         uses: actions/checkout@v6
@@ -309,27 +310,26 @@ jobs:
                 # Force amd64 runner for aptly method due to stability issues on ARM
                 if [[ "${METHOD:-aptly}" == "aptly" ]]; then
                   # Always use amd64 runner for aptly, use QEMU for other arches
-                  runner="ubuntu-latest"
+                  runner='["docker", "X64"]'
                   image_arch="amd64"
                 else
                   # Use architecture-specific runners for other methods (gh, direct)
                   case "${arch}" in
                     amd64)
-                      runner="ubuntu-latest"
-                      #runner="X64"
+                      runner='["docker", "X64"]'
                       image_arch="${arch}"
                       ;;
                     arm64)
-                      runner="ubuntu-24.04-arm"
-                      #runner="docker"
+                      runner='["docker", "ARM64"]'
                       image_arch="${arch}"
                       ;;
                     armhf|riscv64)
-                      runner="ubuntu-latest"
+                      runner='["ubuntu-latest"]'
+                      runner='["docker", "X64"]'
                       image_arch="amd64"  # Use amd64 image with QEMU emulation
                       ;;
                     *)
-                      runner="ubuntu-latest"
+                      runner='["ubuntu-latest"]'
                       image_arch="amd64"
                       ;;
                   esac
@@ -347,7 +347,7 @@ jobs:
                   --arg image_arch "${image_arch}" \
                   --arg registry "ghcr.io/${{ github.repository_owner }}" \
                   --argjson needs_qemu "${needs_qemu}" \
-                  '{name: $name, arch: $arch, release: $release, target: $target, method: $method, install: $install, runner: $runner, image_arch: $image_arch, registry: $registry, needs_qemu: $needs_qemu}'
+                  '{name: $name, arch: $arch, release: $release, target: $target, method: $method, install: $install, runner: ($runner | fromjson), image_arch: $image_arch, registry: $registry, needs_qemu: $needs_qemu}'
               done
             done
           done | jq -s '{"include": .}'
@@ -435,15 +435,20 @@ jobs:
 
           # Install appropriate keyring based on container type
           if grep -q "debian" /etc/os-release; then
-            apt-get -y install debian-keyring
+            apt-get -y install debian-keyring 2>&1 | grep -v "^gpg:" || true
           elif grep -q "ubuntu" /etc/os-release; then
-            apt-get -y install ubuntu-keyring
+            apt-get -y install ubuntu-keyring 2>&1 | grep -v "^gpg:" || true
           fi
 
-          # Import additional keys
-          gpg --no-default-keyring --keyring trustedkeys.gpg \
+          # Import additional keys with timeout and suppress verbose output
+          # Use timeout to prevent hanging on unavailable keyservers
+          if timeout 30 gpg --no-default-keyring --keyring trustedkeys.gpg \
             --keyserver keyserver.ubuntu.com \
-            --recv-keys 648ACFD622F3D138 0E98404D386FA1D9
+            --recv-keys 648ACFD622F3D138 0E98404D386FA1D9 2>&1 | grep -v "not changed\|unchanged\|not checked" || true; then
+            echo "::notice::GPG keys imported successfully"
+          else
+            echo "::warning::Some GPG keys could not be imported from keyserver (non-critical)"
+          fi
 
       - name: "Prepare machine"
         id: preparing
@@ -460,11 +465,18 @@ jobs:
           [[ "${SIGNATURES}" == "ignore" ]] && APTLY_CONF+="-ignore-signatures "
 
           # Read existing releases and create folder structure
+          rm -rf build/output/{debs,debs-beta}
           ALL_RELEASES=($(grep -rw build/config/distributions/*/support -ve 'eos' | cut -d"/" -f4 ))
           for i in ${ALL_RELEASES[@]}; do
             mkdir -p build/output/{debs,debs-beta}/${i}
             mkdir -p build/output/{debs,debs-beta}/extra/${i}-utils
             mkdir -p build/output/{debs,debs-beta}/extra/${i}-desktop
+            # Add example.deb to each subfolder to ensure empty repos are published
+            for repo_root in build/output/debs build/output/debs-beta; do
+              cp build/tools/repository/example.deb ${repo_root}/${i}/
+              cp build/tools/repository/example.deb ${repo_root}/extra/${i}-utils/
+              cp build/tools/repository/example.deb ${repo_root}/extra/${i}-desktop/
+            done
           done
 
           needs_qemu='${{ matrix.needs_qemu }}'
@@ -507,6 +519,7 @@ jobs:
           . os/external/${{ matrix.name }}.conf
 
           SOURCE="temp/"
+          rm -rf ${SOURCE}
           mkdir -p ${SOURCE}
 
           # Get current version from Armbian repository
@@ -521,13 +534,7 @@ jobs:
 
           # Get version from main repository
           BEFORE_VERSION=""
-
-          # Skip version check if SKIP_VERSION_CHECK is enabled (for force repopulation)
-          if [[ "${{ inputs.SKIP_VERSION_CHECK }}" == "true" ]]; then
-            echo "::notice::SKIP_VERSION_CHECK enabled - forcing upload regardless of existing versions"
-            BEFORE_VERSION="0"
-          else
-            # Try main component, desktop component, then extra component
+          # Try main component, desktop component, then extra component
           for repo_component in "main" "${{ matrix.release }}-desktop" "${{ matrix.release }}-utils"; do
             # Build URL for Packages index
             CURRENT_PACKAGES_URL="${PACKAGES_URL}/dists/${{ matrix.release }}/${repo_component}/binary-${{ matrix.arch }}/Packages.gz"
@@ -554,10 +561,9 @@ jobs:
             echo "::warning::Could not find version for $PKG in repository, assuming new package"
             BEFORE_VERSION="0"
           fi
-          fi
 
           echo "BEFORE_VERSION=${BEFORE_VERSION}" >> $GITHUB_OUTPUT
-          
+
           # Download packages based on method
           if [[ ${METHOD} == gh ]]; then
             # GitHub release download method
@@ -833,30 +839,37 @@ jobs:
           # Upload packages directly to remote storage to avoid race conditions
           # This prevents parallel jobs from overwriting each other's packages
           # Note: StrictHostKeychecking=no is used here; consider using proper known_hosts in production
-
+          # Always upload to debs-beta for external packages
           if [[ ${TARGET} == main ]]; then
-            # Upload to main repository directories
-            if grep -qE 'B' <<< "$REPOSITORY"; then
+            # Upload to main repository directory
+            find "$SOURCE" -type f -name "*.deb" -exec \
+              rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \
+              -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs-beta/ \;
+
+            # Upload to debs only if version is newer than what's in repository
+            # This is checked by comparing BEFORE_VERSION from the repository
+            if dpkg --compare-versions "$AFTER_VERSION" gt "$BEFORE_VERSION"; then
+              echo "::notice::Newer version $AFTER_VERSION found, uploading to debs"
               find "$SOURCE" -type f -name "*.deb" -exec \
                 rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \
-                -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:storage/debs-beta/ \;
-            fi
-            if grep -qE 'S' <<< "$REPOSITORY"; then
-              find "$SOURCE" -type f -name "*.deb" -exec \
-                rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \
-                -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:storage/debs/ \;
+                -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs/ \;
+            else
+              echo "::notice::Version $AFTER_VERSION already exists in debs (has $BEFORE_VERSION), skipping upload"
             fi
           else
-            # Upload to specific release directories
-            if grep -qE 'B' <<< "$REPOSITORY"; then
+            # Upload to specific release directory in debs-beta
+            find "$SOURCE" -type f -name "*.deb" -exec \
+              rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \
+              -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs-beta/extra/${{ matrix.release }}-${TARGET}/ \;
+
+            # Upload to debs only if version is newer than what's in repository
+            if dpkg --compare-versions "$AFTER_VERSION" gt "$BEFORE_VERSION"; then
+              echo "::notice::Newer version $AFTER_VERSION found, uploading to debs/extra"
               find "$SOURCE" -type f -name "*.deb" -exec \
                 rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \
-                -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:storage/debs-beta/extra/${{ matrix.release }}-${TARGET}/ \;
-            fi
-            if grep -qE 'S' <<< "$REPOSITORY"; then
-              find "$SOURCE" -type f -name "*.deb" -exec \
-                rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \
-                -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:storage/debs/extra/${{ matrix.release }}-${TARGET}/ \;
+                -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs/extra/${{ matrix.release }}-${TARGET}/ \;
+            else
+              echo "::notice::Version $AFTER_VERSION already exists in debs (has $BEFORE_VERSION), skipping upload"
             fi
           fi
 

.github/workflows/infrastructure-repository-update.yml

@@ -40,8 +40,29 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TEAM: "Release manager"
 
-  Copying:
+  external:
+    name: "Download external"
     needs: Check
+    uses: armbian/armbian.github.io/.github/workflows/infrastructure-download-external.yml@main
+    with:
+      ENABLED: ${{ inputs.download_external != false || github.event.client_payload.download_external != false }}
+      SKIP_VERSION_CHECK: false
+      ACCESS_NAME: armbian
+      BUILD_RUNNER: "docker"
+      HOST_DEPLOY: "repo.armbian.com"
+      PURGE: ${{ inputs.purge_external || false }}
+    secrets:
+      GPG_KEY1: ${{ secrets.GPG_KEY3 }}
+      GPG_KEY2: ${{ secrets.GPG_KEY4 }}
+      ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+      KEY_UPLOAD: ${{ secrets.KEY_UPLOAD }}
+      HOST_UPLOAD: ${{ secrets.HOST_UPLOAD }}
+      HOST_UPLOAD_USER: ${{ secrets.HOST_UPLOAD_USER }}
+      HOST_UPLOAD_PORT: ${{ secrets.HOST_UPLOAD_PORT }}
+      KNOWN_HOSTS_ARMBIAN_UPLOAD: ${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }}
+
+  Copying:
+    needs: external
     runs-on: repository
     steps:
 
@@ -131,31 +152,79 @@ jobs:
               ;;
           esac
 
-  external:
-    name: "Download external"
+          # Always sync external
+          if [ -d "${INCOMING_PATH}/external/debs" ]; then
+              rsync -av \
+              --include='*/' \
+              --include='*.deb' \
+              --exclude='*' \
+              --omit-dir-times --no-perms --no-group \
+              "${INCOMING_PATH}/external/debs/" "${STORAGE_PATH}/debs/" \
+              2>&1 | tee -a "$GITHUB_STEP_SUMMARY" || \
+              echo "Warning: Some files/attrs were not transferred (code 23)" >> "$GITHUB_STEP_SUMMARY"
+          fi
+          if [ -d "${INCOMING_PATH}/external/debs-beta" ]; then
+              rsync -av \
+              --include='*/' \
+              --include='*.deb' \
+              --exclude='*' \
+              --omit-dir-times --no-perms --no-group \
+              "${INCOMING_PATH}/external/debs-beta/" "${STORAGE_PATH}/debs-beta/" \
+              2>&1 | tee -a "$GITHUB_STEP_SUMMARY" || \
+              echo "Warning: Some files/attrs were not transferred (code 23)" >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+  prepare-repos:
+    name: "Prepare repositories"
     needs: Copying
-    uses: armbian/armbian.github.io/.github/workflows/infrastructure-download-external.yml@main
-    with:
-      ENABLED: ${{ inputs.download_external != false || github.event.client_payload.download_external != false }}
-      SKIP_VERSION_CHECK: false
-      ACCESS_NAME: armbian
-      BUILD_RUNNER: "ubuntu-latest"
-      HOST_DEPLOY: "repo.armbian.com"
-      PURGE: ${{ inputs.purge_external || false }}
-    secrets:
-      GPG_KEY1: ${{ secrets.GPG_KEY3 }}
-      GPG_KEY2: ${{ secrets.GPG_KEY4 }}
-      ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
-      KEY_UPLOAD: ${{ secrets.KEY_UPLOAD }}
-      HOST_UPLOAD: ${{ secrets.HOST_UPLOAD }}
-      HOST_UPLOAD_USER: ${{ secrets.HOST_UPLOAD_USER }}
-      HOST_UPLOAD_PORT: ${{ secrets.HOST_UPLOAD_PORT }}
-      KNOWN_HOSTS_ARMBIAN_UPLOAD: ${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }}
+    runs-on: repository
+    steps:
+      - name: Checkout build repository
+        uses: actions/checkout@v6
+        with:
+          repository: armbian/build
+          path: build
+          clean: false
+
+      - name: Checkout armbian.github.io repository
+        uses: actions/checkout@v6
+        with:
+          repository: armbian/armbian.github.io
+          path: armbian-github-io
+          clean: false
+
+      - name: Copy example.deb to repository subfolders
+        run: |
+
+          # Get all supported releases from build config
+          ALL_RELEASES=($(grep -rw build/config/distributions/*/support -ve 'eos' | cut -d"/" -f4))
+          echo "Found ${#ALL_RELEASES[@]} releases: ${ALL_RELEASES[*]}" | tee -a $GITHUB_STEP_SUMMARY
+
+
+          sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/$RELEASE/"
+          sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/$RELEASE/"
 
+          # Copy example.deb to all subfolders for debs
+          for RELEASE in "${ALL_RELEASES[@]}"; do
+            echo "Copying example.deb for debs/$RELEASE and subfolders" | tee -a $GITHUB_STEP_SUMMARY
+            mkdir -p "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-utils"
+            mkdir -p "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-desktop"
+            sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-utils/"
+            sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-desktop/"
+          done
+
+          # Copy example.deb to all subfolders for debs-beta
+          for RELEASE in "${ALL_RELEASES[@]}"; do
+            echo "Copying example.deb for debs-beta/$RELEASE and subfolders" | tee -a $GITHUB_STEP_SUMMARY
+            mkdir -p "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-utils"
+            mkdir -p "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-desktop"
+            sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-utils/"
+            sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-desktop/"
+          done
 
   fix-permissions:
     name: "Fix permissions"
-    needs: external
+    needs: prepare-repos
     runs-on: repository
     strategy:
       matrix:
@@ -333,7 +402,7 @@ jobs:
         with:
           repository: armbian/build
           fetch-depth: 1
-          ref: main
+          ref: always
           clean: false
 
       - name: "Build repository ${{ matrix.repository.name }}"
@@ -412,7 +481,7 @@ jobs:
         with:
           repository: armbian/build
           clean: false
-          ref: main
+          ref: always
           fetch-depth: 1
           path: build
 
@@ -534,7 +603,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           repository: armbian/build
-          ref: main
+          ref: always
           fetch-depth: 1
           clean: false
 
@@ -646,7 +715,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           repository: armbian/build
-          ref: main
+          ref: always
           fetch-depth: 1
           clean: false