From 881b4660bbc601f928bccb416831e2fa66cfee09 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Fri, 9 Jan 2026 18:07:36 +0100 Subject: [PATCH 01/11] Implement new parallel repository workflow in GitHub Actions Updated infrastructure-repository-update.yml to use the new 3-step parallel repository architecture that eliminates duplicate work. Workflow changes: - Step 1: main job builds common component with update-main - Step 2: postclean jobs build release-specific components in parallel using -R flag (isolated DBs, no publishing) - Step 3: NEW merge job combines common + release-specific components and publishes complete repositories - Updated cleanup job dependency to wait for merge Benefits: - Common packages added once instead of per-release - True parallelism across multiple releases - Better error isolation between releases - Scalable architecture for adding more releases Signed-off-by: Igor Pecovnik --- .../infrastructure-repository-update.yml | 105 +++++++++++++++++- 1 file changed, 102 insertions(+), 3 deletions(-) diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index 4821f4049..97bb293fe 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -581,9 +581,10 @@ jobs: INPUT_DIR="${STORAGE_PATH}/${PACKAGE}" OUTPUT_DIR="${PUBLISH_PATH}-${PACKAGE}" - echo "### Checking for .deb files in $PACKAGE/$RELEASE" | tee -a "$GITHUB_STEP_SUMMARY" + echo "### Building release-specific components for $PACKAGE/$RELEASE" | tee -a "$GITHUB_STEP_SUMMARY" echo "" echo "Input directory: $INPUT_DIR" | tee -a "$GITHUB_STEP_SUMMARY" + echo "Output directory: $OUTPUT_DIR" | tee -a "$GITHUB_STEP_SUMMARY" echo "" # Validate input directory exists @@ -599,20 +600,118 @@ jobs: # Only run repo.sh if there are files to process if [ "${DEB_COUNT}" -gt 0 ]; then - echo "Running repository update for $RELEASE..." | tee -a "$GITHUB_STEP_SUMMARY" + echo "Building release-specific repos and snapshots for $RELEASE..." | tee -a "$GITHUB_STEP_SUMMARY" + echo "Note: Common component was already built by 'main' job" | tee -a "$GITHUB_STEP_SUMMARY" + echo "Release-specific components will be merged by 'merge' job" | tee -a "$GITHUB_STEP_SUMMARY" + echo "" tools/repository/repo.sh \ -c update \ -R "$RELEASE" \ -k \ -i "$INPUT_DIR" \ -o "$OUTPUT_DIR" + echo "✓ Release-specific components built for $RELEASE" | tee -a "$GITHUB_STEP_SUMMARY" else echo "::notice::No .deb files found, skipping repository update" fi + merge: + needs: postclean + strategy: + fail-fast: false + matrix: + repository: + - name: debs + - name: debs-beta + name: "Merge ${{ matrix.repository.name }}" + timeout-minutes: 120 + runs-on: repository + steps: + + # Cleaning self hosted runners + - name: Runner clean + uses: armbian/actions/runner-clean@main + + - name: Import GPG key + uses: crazy-max/ghaction-import-gpg@v6 + with: + gpg_private_key: ${{ secrets.GPG_KEY3 }} + + - name: Import GPG key + uses: crazy-max/ghaction-import-gpg@v6 + with: + gpg_private_key: ${{ secrets.GPG_KEY4 }} + + - name: Checkout build repository + uses: actions/checkout@v6 + with: + repository: armbian/build + ref: main + fetch-depth: 1 + clean: false + + - name: "Merge repository ${{ matrix.repository.name }}" + shell: bash + run: | + set -e + set -o pipefail + + STORAGE_PATH="${{ env.STORAGE_PATH }}" + PUBLISH_PATH="${{ env.PUBLISHING_PATH }}" + REPO_NAME="${{ matrix.repository.name }}" + + # Validate paths + case "$STORAGE_PATH" in + *"/armbian/openssh-server/storage"*) + ;; + *) + echo "::error::Invalid storage path: $STORAGE_PATH" + exit 1 + ;; + esac + + case "$PUBLISH_PATH" in + *"/publishing/repository"*) + ;; + *) + echo "::error::Invalid publishing path: $PUBLISH_PATH" + exit 1 + ;; + esac + + INPUT_DIR="${STORAGE_PATH}/${REPO_NAME}" + OUTPUT_DIR="${PUBLISH_PATH}-${REPO_NAME}" + + echo "### Merging repository $REPO_NAME" | tee -a "$GITHUB_STEP_SUMMARY" + echo "" + echo "Input directory: $INPUT_DIR" | tee -a "$GITHUB_STEP_SUMMARY" + echo "Output directory: $OUTPUT_DIR" | tee -a "$GITHUB_STEP_SUMMARY" + echo "" + + # Validate output directory exists + if [ ! -d "$OUTPUT_DIR" ]; then + echo "::notice::Output directory does not exist: $OUTPUT_DIR" + echo "This is expected if no packages were processed yet" | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + echo "Merging common component with release-specific components..." | tee -a "$GITHUB_STEP_SUMMARY" + echo "This imports from isolated databases and publishes complete repositories" | tee -a "$GITHUB_STEP_SUMMARY" + echo "" + + # Run merge command to combine common + release-specific components + tools/repository/repo.sh \ + -c merge \ + -i "$INPUT_DIR" \ + -o "$OUTPUT_DIR" + + echo "✓ Repository merge completed for $REPO_NAME" | tee -a "$GITHUB_STEP_SUMMARY" + echo "" + echo "Repository is now ready with all components combined" | tee -a "$GITHUB_STEP_SUMMARY" + cleanup: name: "Clean input" - needs: postclean + needs: merge runs-on: repository strategy: matrix: From 7f06a08d3016e5f9e88d4fe12a70bd25b91c69b4 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Fri, 9 Jan 2026 18:10:08 +0100 Subject: [PATCH 02/11] Use dev branch --- .github/workflows/infrastructure-repository-update.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index 97bb293fe..7ccad9b96 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -333,7 +333,7 @@ jobs: with: repository: armbian/build fetch-depth: 1 - ref: main + ref: feature/new-parallel-repo-architecture clean: false - name: "Build repository ${{ matrix.repository.name }}" @@ -412,7 +412,7 @@ jobs: with: repository: armbian/build clean: false - ref: main + ref: feature/new-parallel-repo-architecture fetch-depth: 1 path: build @@ -534,7 +534,7 @@ jobs: uses: actions/checkout@v6 with: repository: armbian/build - ref: main + ref: feature/new-parallel-repo-architecture fetch-depth: 1 clean: false @@ -646,7 +646,7 @@ jobs: uses: actions/checkout@v6 with: repository: armbian/build - ref: main + ref: feature/new-parallel-repo-architecture fetch-depth: 1 clean: false From fe2cf8f2e1c610c2f12559356239668b422df373 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Sun, 11 Jan 2026 00:09:23 +0100 Subject: [PATCH 03/11] Revert "Use dev branch" This reverts commit 7f06a08d3016e5f9e88d4fe12a70bd25b91c69b4. --- .github/workflows/infrastructure-repository-update.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index 7ccad9b96..97bb293fe 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -333,7 +333,7 @@ jobs: with: repository: armbian/build fetch-depth: 1 - ref: feature/new-parallel-repo-architecture + ref: main clean: false - name: "Build repository ${{ matrix.repository.name }}" @@ -412,7 +412,7 @@ jobs: with: repository: armbian/build clean: false - ref: feature/new-parallel-repo-architecture + ref: main fetch-depth: 1 path: build @@ -534,7 +534,7 @@ jobs: uses: actions/checkout@v6 with: repository: armbian/build - ref: feature/new-parallel-repo-architecture + ref: main fetch-depth: 1 clean: false @@ -646,7 +646,7 @@ jobs: uses: actions/checkout@v6 with: repository: armbian/build - ref: feature/new-parallel-repo-architecture + ref: main fetch-depth: 1 clean: false From ce49644d01613b8557786b040a897363be4ec52c Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Sun, 11 Jan 2026 10:40:45 +0100 Subject: [PATCH 04/11] Change order --- .../infrastructure-download-external.yml | 1 + .../infrastructure-repository-update.yml | 44 +++++++++---------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/.github/workflows/infrastructure-download-external.yml b/.github/workflows/infrastructure-download-external.yml index 0973e1a64..26aa39f6b 100644 --- a/.github/workflows/infrastructure-download-external.yml +++ b/.github/workflows/infrastructure-download-external.yml @@ -65,6 +65,7 @@ env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.number }} PACKAGES_URL: "https://fi.mirror.armbian.de/apt" + DOWNLOAD_PATH: /armbian/openssh-server/storage jobs: diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index 97bb293fe..c2b0a4acb 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -40,8 +40,29 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TEAM: "Release manager" - Copying: + external: + name: "Download external" needs: Check + uses: armbian/armbian.github.io/.github/workflows/infrastructure-download-external.yml@main + with: + ENABLED: ${{ inputs.download_external != false || github.event.client_payload.download_external != false }} + SKIP_VERSION_CHECK: false + ACCESS_NAME: armbian + BUILD_RUNNER: "ubuntu-latest" + HOST_DEPLOY: "repo.armbian.com" + PURGE: ${{ inputs.purge_external || false }} + secrets: + GPG_KEY1: ${{ secrets.GPG_KEY3 }} + GPG_KEY2: ${{ secrets.GPG_KEY4 }} + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + KEY_UPLOAD: ${{ secrets.KEY_UPLOAD }} + HOST_UPLOAD: ${{ secrets.HOST_UPLOAD }} + HOST_UPLOAD_USER: ${{ secrets.HOST_UPLOAD_USER }} + HOST_UPLOAD_PORT: ${{ secrets.HOST_UPLOAD_PORT }} + KNOWN_HOSTS_ARMBIAN_UPLOAD: ${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }} + + Copying: + needs: external runs-on: repository steps: @@ -131,27 +152,6 @@ jobs: ;; esac - external: - name: "Download external" - needs: Copying - uses: armbian/armbian.github.io/.github/workflows/infrastructure-download-external.yml@main - with: - ENABLED: ${{ inputs.download_external != false || github.event.client_payload.download_external != false }} - SKIP_VERSION_CHECK: false - ACCESS_NAME: armbian - BUILD_RUNNER: "ubuntu-latest" - HOST_DEPLOY: "repo.armbian.com" - PURGE: ${{ inputs.purge_external || false }} - secrets: - GPG_KEY1: ${{ secrets.GPG_KEY3 }} - GPG_KEY2: ${{ secrets.GPG_KEY4 }} - ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} - KEY_UPLOAD: ${{ secrets.KEY_UPLOAD }} - HOST_UPLOAD: ${{ secrets.HOST_UPLOAD }} - HOST_UPLOAD_USER: ${{ secrets.HOST_UPLOAD_USER }} - HOST_UPLOAD_PORT: ${{ secrets.HOST_UPLOAD_PORT }} - KNOWN_HOSTS_ARMBIAN_UPLOAD: ${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }} - fix-permissions: name: "Fix permissions" From 3d9702ba87d9713f398e0b33c795bc9910720805 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Sun, 11 Jan 2026 10:41:22 +0100 Subject: [PATCH 05/11] Use testing branch --- .github/workflows/infrastructure-repository-update.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index c2b0a4acb..58ac5ff1f 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -333,7 +333,7 @@ jobs: with: repository: armbian/build fetch-depth: 1 - ref: main + ref: always clean: false - name: "Build repository ${{ matrix.repository.name }}" @@ -412,7 +412,7 @@ jobs: with: repository: armbian/build clean: false - ref: main + ref: always fetch-depth: 1 path: build @@ -534,7 +534,7 @@ jobs: uses: actions/checkout@v6 with: repository: armbian/build - ref: main + ref: always fetch-depth: 1 clean: false @@ -646,7 +646,7 @@ jobs: uses: actions/checkout@v6 with: repository: armbian/build - ref: main + ref: always fetch-depth: 1 clean: false From 1cde315691185d7a4c60c1c8d80b6f417768e3ce Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Sun, 11 Jan 2026 11:15:05 +0100 Subject: [PATCH 06/11] fix: use UPLOAD_PATH env var and add example.deb to empty repos - Replace DOWNLOAD_PATH with UPLOAD_PATH environment variable - Update UPLOAD_PATH to "storage/incoming/external" for better organization - Update all rsync commands to use ${UPLOAD_PATH} instead of hardcoded "storage/" - Add example.deb to each generated subfolder (main, utils, desktop) to ensure empty repos are properly published with complete structure This ensures repository components are always publishable even when no packages are added to them. Signed-off-by: Igor Pecovnik --- .../infrastructure-download-external.yml | 18 ++++++++++++------ .../infrastructure-repository-update.yml | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/infrastructure-download-external.yml b/.github/workflows/infrastructure-download-external.yml index 26aa39f6b..c90c32791 100644 --- a/.github/workflows/infrastructure-download-external.yml +++ b/.github/workflows/infrastructure-download-external.yml @@ -65,7 +65,7 @@ env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.number }} PACKAGES_URL: "https://fi.mirror.armbian.de/apt" - DOWNLOAD_PATH: /armbian/openssh-server/storage + UPLOAD_PATH: "storage/incoming/external" jobs: @@ -220,7 +220,7 @@ jobs: # Note: StrictHostKeychecking=no is used here; consider using proper known_hosts in production ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }} \ ${{ inputs.HOST_USER }}@${{ inputs.HOST_DEPLOY }} \ - "rm -rf storage/artifacts/*" + "rm -rf ${UPLOAD_PATH}/artifacts/*" - name: Checkout Armbian OS repository uses: actions/checkout@v6 @@ -466,6 +466,12 @@ jobs: mkdir -p build/output/{debs,debs-beta}/${i} mkdir -p build/output/{debs,debs-beta}/extra/${i}-utils mkdir -p build/output/{debs,debs-beta}/extra/${i}-desktop + # Add example.deb to each subfolder to ensure empty repos are published + for repo_root in build/output/debs build/output/debs-beta; do + cp build/tools/repository/example.deb ${repo_root}/${i}/ + cp build/tools/repository/example.deb ${repo_root}/extra/${i}-utils/ + cp build/tools/repository/example.deb ${repo_root}/extra/${i}-desktop/ + done done needs_qemu='${{ matrix.needs_qemu }}' @@ -840,24 +846,24 @@ jobs: if grep -qE 'B' <<< "$REPOSITORY"; then find "$SOURCE" -type f -name "*.deb" -exec \ rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ - -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:storage/debs-beta/ \; + -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs-beta/ \; fi if grep -qE 'S' <<< "$REPOSITORY"; then find "$SOURCE" -type f -name "*.deb" -exec \ rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ - -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:storage/debs/ \; + -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs/ \; fi else # Upload to specific release directories if grep -qE 'B' <<< "$REPOSITORY"; then find "$SOURCE" -type f -name "*.deb" -exec \ rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ - -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:storage/debs-beta/extra/${{ matrix.release }}-${TARGET}/ \; + -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs-beta/extra/${{ matrix.release }}-${TARGET}/ \; fi if grep -qE 'S' <<< "$REPOSITORY"; then find "$SOURCE" -type f -name "*.deb" -exec \ rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ - -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:storage/debs/extra/${{ matrix.release }}-${TARGET}/ \; + -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs/extra/${{ matrix.release }}-${TARGET}/ \; fi fi diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index 58ac5ff1f..917b411d8 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -43,7 +43,7 @@ jobs: external: name: "Download external" needs: Check - uses: armbian/armbian.github.io/.github/workflows/infrastructure-download-external.yml@main + uses: armbian/armbian.github.io/.github/workflows/infrastructure-download-external.yml@always with: ENABLED: ${{ inputs.download_external != false || github.event.client_payload.download_external != false }} SKIP_VERSION_CHECK: false From dd8e7c052aa5b069527cd626fbb9e13bdf5e7965 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Sun, 11 Jan 2026 11:44:58 +0100 Subject: [PATCH 07/11] Update --- .../infrastructure-download-external.yml | 24 +++++++++---------- .../infrastructure-repository-update.yml | 8 +++---- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/infrastructure-download-external.yml b/.github/workflows/infrastructure-download-external.yml index c90c32791..8af5cb6a9 100644 --- a/.github/workflows/infrastructure-download-external.yml +++ b/.github/workflows/infrastructure-download-external.yml @@ -310,27 +310,25 @@ jobs: # Force amd64 runner for aptly method due to stability issues on ARM if [[ "${METHOD:-aptly}" == "aptly" ]]; then # Always use amd64 runner for aptly, use QEMU for other arches - runner="ubuntu-latest" + runner='["docker", "X64"]' image_arch="amd64" else # Use architecture-specific runners for other methods (gh, direct) case "${arch}" in amd64) - runner="ubuntu-latest" - #runner="X64" + runner='["docker", "X64"]' image_arch="${arch}" ;; arm64) - runner="ubuntu-24.04-arm" - #runner="docker" + runner='["docker", "ARM64"]' image_arch="${arch}" ;; armhf|riscv64) - runner="ubuntu-latest" + runner='["ubuntu-latest"]' image_arch="amd64" # Use amd64 image with QEMU emulation ;; *) - runner="ubuntu-latest" + runner='["ubuntu-latest"]' image_arch="amd64" ;; esac @@ -348,7 +346,7 @@ jobs: --arg image_arch "${image_arch}" \ --arg registry "ghcr.io/${{ github.repository_owner }}" \ --argjson needs_qemu "${needs_qemu}" \ - '{name: $name, arch: $arch, release: $release, target: $target, method: $method, install: $install, runner: $runner, image_arch: $image_arch, registry: $registry, needs_qemu: $needs_qemu}' + '{name: $name, arch: $arch, release: $release, target: $target, method: $method, install: $install, runner: ($runner | fromjson), image_arch: $image_arch, registry: $registry, needs_qemu: $needs_qemu}' done done done | jq -s '{"include": .}' @@ -461,6 +459,7 @@ jobs: [[ "${SIGNATURES}" == "ignore" ]] && APTLY_CONF+="-ignore-signatures " # Read existing releases and create folder structure + rm -rf build/output/{debs,debs-beta} ALL_RELEASES=($(grep -rw build/config/distributions/*/support -ve 'eos' | cut -d"/" -f4 )) for i in ${ALL_RELEASES[@]}; do mkdir -p build/output/{debs,debs-beta}/${i} @@ -514,6 +513,7 @@ jobs: . os/external/${{ matrix.name }}.conf SOURCE="temp/" + rm -rf ${SOURCE} mkdir -p ${SOURCE} # Get current version from Armbian repository @@ -846,24 +846,24 @@ jobs: if grep -qE 'B' <<< "$REPOSITORY"; then find "$SOURCE" -type f -name "*.deb" -exec \ rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ - -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs-beta/ \; + -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs-beta/ \; fi if grep -qE 'S' <<< "$REPOSITORY"; then find "$SOURCE" -type f -name "*.deb" -exec \ rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ - -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs/ \; + -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs/ \; fi else # Upload to specific release directories if grep -qE 'B' <<< "$REPOSITORY"; then find "$SOURCE" -type f -name "*.deb" -exec \ rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ - -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs-beta/extra/${{ matrix.release }}-${TARGET}/ \; + -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs-beta/extra/${{ matrix.release }}-${TARGET}/ \; fi if grep -qE 'S' <<< "$REPOSITORY"; then find "$SOURCE" -type f -name "*.deb" -exec \ rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ - -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs/extra/${{ matrix.release }}-${TARGET}/ \; + -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs/extra/${{ matrix.release }}-${TARGET}/ \; fi fi diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index 917b411d8..f1ad42610 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -22,9 +22,9 @@ env: CLEANUP_INPUT: true DRY_RUN_SYNC: false -concurrency: - group: pipeline - cancel-in-progress: false +#concurrency: +# group: pipeline +# cancel-in-progress: false jobs: @@ -48,7 +48,7 @@ jobs: ENABLED: ${{ inputs.download_external != false || github.event.client_payload.download_external != false }} SKIP_VERSION_CHECK: false ACCESS_NAME: armbian - BUILD_RUNNER: "ubuntu-latest" + BUILD_RUNNER: "docker" HOST_DEPLOY: "repo.armbian.com" PURGE: ${{ inputs.purge_external || false }} secrets: From 6d25cf58ff22c0ac9a9b5ae725e3aaafae609600 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Sun, 11 Jan 2026 15:06:03 +0100 Subject: [PATCH 08/11] Implement skip downloading if file exist on server --- .../infrastructure-download-external.yml | 101 ++++++++++++++---- .../infrastructure-repository-update.yml | 5 + 2 files changed, 88 insertions(+), 18 deletions(-) diff --git a/.github/workflows/infrastructure-download-external.yml b/.github/workflows/infrastructure-download-external.yml index 8af5cb6a9..47d25a352 100644 --- a/.github/workflows/infrastructure-download-external.yml +++ b/.github/workflows/infrastructure-download-external.yml @@ -434,15 +434,20 @@ jobs: # Install appropriate keyring based on container type if grep -q "debian" /etc/os-release; then - apt-get -y install debian-keyring + apt-get -y install debian-keyring 2>&1 | grep -v "^gpg:" || true elif grep -q "ubuntu" /etc/os-release; then - apt-get -y install ubuntu-keyring + apt-get -y install ubuntu-keyring 2>&1 | grep -v "^gpg:" || true fi - # Import additional keys - gpg --no-default-keyring --keyring trustedkeys.gpg \ + # Import additional keys with timeout and suppress verbose output + # Use timeout to prevent hanging on unavailable keyservers + if timeout 30 gpg --no-default-keyring --keyring trustedkeys.gpg \ --keyserver keyserver.ubuntu.com \ - --recv-keys 648ACFD622F3D138 0E98404D386FA1D9 + --recv-keys 648ACFD622F3D138 0E98404D386FA1D9 2>&1 | grep -v "not changed\|unchanged\|not checked" || true; then + echo "::notice::GPG keys imported successfully" + else + echo "::warning::Some GPG keys could not be imported from keyserver (non-critical)" + fi - name: "Prepare machine" id: preparing @@ -526,14 +531,49 @@ jobs: COMPONENT="extra/${{ matrix.release }}-${TARGET}" fi - # Get version from main repository - BEFORE_VERSION="" + # Check if files already exist on remote storage to avoid re-downloading + # Only check if SKIP_VERSION_CHECK is not enabled + SKIP_DOWNLOAD=false + if [[ "${{ inputs.SKIP_VERSION_CHECK }}" != "true" ]]; then + echo "::debug::Checking if files already exist on remote storage..." + # Determine remote path based on REPOSITORY (B=beta, S=stable) + REMOTE_BASE_PATH="${UPLOAD_PATH}" + if grep -qE 'B' <<< "$REPOSITORY"; then + if [[ "${TARGET}" == "main" ]]; then + REMOTE_PATH="debs-beta/" + else + REMOTE_PATH="debs-beta/extra/${{ matrix.release }}-${TARGET}/" + fi + elif grep -qE 'S' <<< "$REPOSITORY"; then + if [[ "${TARGET}" == "main" ]]; then + REMOTE_PATH="debs/" + else + REMOTE_PATH="debs/extra/${{ matrix.release }}-${TARGET}/" + fi + fi - # Skip version check if SKIP_VERSION_CHECK is enabled (for force repopulation) - if [[ "${{ inputs.SKIP_VERSION_CHECK }}" == "true" ]]; then - echo "::notice::SKIP_VERSION_CHECK enabled - forcing upload regardless of existing versions" - BEFORE_VERSION="0" - else + # Check if any .deb files for this package exist remotely + REMOTE_FILES=$(ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }} \ + ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }} \ + "find ${REMOTE_BASE_PATH}/${REMOTE_PATH} -type f -name '${PKG}*.deb' 2>/dev/null" || true) + + if [[ -n "$REMOTE_FILES" ]]; then + echo "::notice::Found existing files on remote storage for $PKG:" + echo "$REMOTE_FILES" + # Get the version from the newest remote file + REMOTE_VERSION=$(echo "$REMOTE_FILES" | xargs -I {} basename {} | \ + sed "s/${PKG}_//" | sed "s/_.*//" | sort -V | tail -n 1) + echo "::notice::Remote version: $REMOTE_VERSION" + BEFORE_VERSION="$REMOTE_VERSION" + SKIP_DOWNLOAD=true + else + echo "::debug::No existing files found on remote storage, will proceed with download" + fi + fi + + # Get version from main repository (only if not found on remote) + if [[ -z "$BEFORE_VERSION" ]]; then + BEFORE_VERSION="" # Try main component, desktop component, then extra component for repo_component in "main" "${{ matrix.release }}-desktop" "${{ matrix.release }}-utils"; do # Build URL for Packages index @@ -564,7 +604,16 @@ jobs: fi echo "BEFORE_VERSION=${BEFORE_VERSION}" >> $GITHUB_OUTPUT - + echo "SKIP_DOWNLOAD=${SKIP_DOWNLOAD}" >> $GITHUB_OUTPUT + + # Skip download if files already exist on remote and version check passed + if [[ "$SKIP_DOWNLOAD" == "true" ]]; then + echo "::notice::Skipping download - files already exist on remote storage with version $BEFORE_VERSION" + # Create a dummy file to satisfy subsequent steps + touch "${SOURCE}skip.txt" + else + echo "::notice::Proceeding with download..." + # Download packages based on method if [[ ${METHOD} == gh ]]; then # GitHub release download method @@ -806,9 +855,10 @@ jobs: done < <(find "$root" -type f -name '*.deb' -printf '%h\0' | sort -zu) done fi + fi # End of SKIP_DOWNLOAD check # Repack deb files with Armbian version - if [[ "${{ matrix.name }}" == "firefox" ]]; then + if [[ "${{ matrix.name }}" == "firefox" ]] && [[ "$SKIP_DOWNLOAD" != "true" ]]; then apt-get install -y devscripts DEBS=($(sudo find "${SOURCE}"* -type f -name '*firefox_*.deb')) for d in "${DEBS[@]}"; do @@ -823,11 +873,17 @@ jobs: fi # Store info to GitHub Actions - AFTER_VERSION=$(find "$SOURCE" -type f -name "${INSTALL%% *}*.deb" -exec dpkg-deb -f {} Version \; | sort | uniq | tail -n 1 | cut -d":" -f2) - if [[ -z $AFTER_VERSION ]]; then - AFTER_VERSION=$(find "$SOURCE" -type f -name "*.deb" -exec dpkg-deb -f {} Version \; | sort | uniq | tail -n 1 | cut -d":" -f2) + if [[ "$SKIP_DOWNLOAD" == "true" ]]; then + # Use the remote version as AFTER_VERSION since we skipped download + AFTER_VERSION="$BEFORE_VERSION" + PKG_LINES="Skipped download - using existing files on remote storage" + else + AFTER_VERSION=$(find "$SOURCE" -type f -name "${INSTALL%% *}*.deb" -exec dpkg-deb -f {} Version \; | sort | uniq | tail -n 1 | cut -d":" -f2) + if [[ -z $AFTER_VERSION ]]; then + AFTER_VERSION=$(find "$SOURCE" -type f -name "*.deb" -exec dpkg-deb -f {} Version \; | sort | uniq | tail -n 1 | cut -d":" -f2) + fi + PKG_LINES="$(find "$SOURCE" -type f -name "*.deb" -printf "%f\n" 2>/dev/null | sort | sed 's/$/
/')" fi - PKG_LINES="$(find "$SOURCE" -type f -name "*.deb" -printf "%f\n" 2>/dev/null | sort | sed 's/$/
/')" echo "AFTER_VERSION=${AFTER_VERSION}" >> $GITHUB_OUTPUT # Determine if update is needed @@ -841,6 +897,8 @@ jobs: # This prevents parallel jobs from overwriting each other's packages # Note: StrictHostKeychecking=no is used here; consider using proper known_hosts in production + # Skip upload if download was skipped (files already exist on remote) + if [[ "$SKIP_DOWNLOAD" != "true" ]]; then if [[ ${TARGET} == main ]]; then # Upload to main repository directories if grep -qE 'B' <<< "$REPOSITORY"; then @@ -866,6 +924,7 @@ jobs: -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs/extra/${{ matrix.release }}-${TARGET}/ \; fi fi + fi # End of SKIP_DOWNLOAD upload check # Upload to repository if version changed if dpkg --compare-versions "$AFTER_VERSION" gt "$BEFORE_VERSION"; then @@ -876,6 +935,12 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY echo "packages:
$PKG_LINES" >> $GITHUB_STEP_SUMMARY + elif [[ "$SKIP_DOWNLOAD" == "true" ]]; then + # Add summary for skipped downloads + echo '| name | method | arch | release | needs_qemu | target | version | status |' >> $GITHUB_STEP_SUMMARY + echo '|--------|-------|------|---------|------------|--------|---------|--------|' >> $GITHUB_STEP_SUMMARY + echo "| ${{ matrix.name }} | ${{ matrix.method }} | ${{ matrix.arch }} | ${{ matrix.release }} | ${{ matrix.needs_qemu }} | ${{ matrix.target }} | $BEFORE_VERSION | ⏭️ Skipped (already on remote) |" >> $GITHUB_STEP_SUMMARY + elif [[ "${{ inputs.HIDE_NO_UPDATE }}" == "true" ]]; then # Exit if HIDE_NO_UPDATE is enabled and no update needed echo "::notice::No update needed for ${{ matrix.name }} on ${{ matrix.arch }}, exiting early" diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index f1ad42610..7e8b733c3 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -152,6 +152,11 @@ jobs: ;; esac + # Always sync external + if [ -d "${INCOMING_PATH}/external" ]; then + rsync -av --omit-dir-times --no-perms --no-group ${INCOMING_PATH}/external/debs ${STORAGE_PATH}/debs 2>&1 | tee -a $GITHUB_STEP_SUMMARY || echo "Warning: Some files/attrs were not transferred (code 23)" >> $GITHUB_STEP_SUMMARY + rsync -av --omit-dir-times --no-perms --no-group ${INCOMING_PATH}/external/debs-beta ${STORAGE_PATH}/debs-beta 2>&1 | tee -a $GITHUB_STEP_SUMMARY || echo "Warning: Some files/attrs were not transferred (code 23)" >> $GITHUB_STEP_SUMMARY + fi fix-permissions: name: "Fix permissions" From 592b4d81924cb24b24fd08dd6bd37824e2ae1751 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Sun, 11 Jan 2026 16:59:30 +0100 Subject: [PATCH 09/11] feat: add architecture filtering to remote package check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add architecture-aware filtering when checking for existing packages on remote storage to prevent skipping downloads for wrong architectures. - Map Debian arch names to common filename variants (amd64→x86_64,x64, arm64→aarch64, etc.) - Build find command with architecture-specific patterns - Only skip download if package exists for matching architecture Signed-off-by: Igor Pecovnik --- .../infrastructure-download-external.yml | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/infrastructure-download-external.yml b/.github/workflows/infrastructure-download-external.yml index 47d25a352..9ae09d5f5 100644 --- a/.github/workflows/infrastructure-download-external.yml +++ b/.github/workflows/infrastructure-download-external.yml @@ -552,13 +552,39 @@ jobs: fi fi - # Check if any .deb files for this package exist remotely + # Check if any .deb files for this package exist remotely with matching architecture + # Map Debian architecture names to common variants used in .deb filenames + case "${{ matrix.arch }}" in + amd64) + ARCH_PATTERNS=("amd64" "x86_64" "x64") + ;; + arm64) + ARCH_PATTERNS=("arm64" "aarch64") + ;; + armhf) + ARCH_PATTERNS=("armhf" "armv7h" "armv7l" "armv6") + ;; + riscv64) + ARCH_PATTERNS=("riscv64") + ;; + *) + ARCH_PATTERNS=("${{ matrix.arch }}") + ;; + esac + + # Build find command with architecture-specific patterns + FIND_CMD="find ${REMOTE_BASE_PATH}/${REMOTE_PATH} -type f -name '${PKG}_*.deb'" + for arch_pat in "${ARCH_PATTERNS[@]}"; do + FIND_CMD="$FIND_CMD -o -name '*_${arch_pat}.deb'" + done + FIND_CMD="$FIND_CMD 2>/dev/null" + REMOTE_FILES=$(ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }} \ ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }} \ - "find ${REMOTE_BASE_PATH}/${REMOTE_PATH} -type f -name '${PKG}*.deb' 2>/dev/null" || true) + "$FIND_CMD" || true) if [[ -n "$REMOTE_FILES" ]]; then - echo "::notice::Found existing files on remote storage for $PKG:" + echo "::notice::Found existing files on remote storage for $PKG (arch: ${{ matrix.arch }}):" echo "$REMOTE_FILES" # Get the version from the newest remote file REMOTE_VERSION=$(echo "$REMOTE_FILES" | xargs -I {} basename {} | \ @@ -567,7 +593,7 @@ jobs: BEFORE_VERSION="$REMOTE_VERSION" SKIP_DOWNLOAD=true else - echo "::debug::No existing files found on remote storage, will proceed with download" + echo "::debug::No existing files found on remote storage for $PKG (arch: ${{ matrix.arch }}), will proceed with download" fi fi From 633b04d523021bea2d18032652cd67073d5b38e0 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Sun, 11 Jan 2026 17:13:42 +0100 Subject: [PATCH 10/11] Drop caching --- .../infrastructure-download-external.yml | 110 ++---------------- .../infrastructure-repository-update.yml | 45 +++++++ 2 files changed, 53 insertions(+), 102 deletions(-) diff --git a/.github/workflows/infrastructure-download-external.yml b/.github/workflows/infrastructure-download-external.yml index 9ae09d5f5..5b741b007 100644 --- a/.github/workflows/infrastructure-download-external.yml +++ b/.github/workflows/infrastructure-download-external.yml @@ -531,76 +531,9 @@ jobs: COMPONENT="extra/${{ matrix.release }}-${TARGET}" fi - # Check if files already exist on remote storage to avoid re-downloading - # Only check if SKIP_VERSION_CHECK is not enabled - SKIP_DOWNLOAD=false - if [[ "${{ inputs.SKIP_VERSION_CHECK }}" != "true" ]]; then - echo "::debug::Checking if files already exist on remote storage..." - # Determine remote path based on REPOSITORY (B=beta, S=stable) - REMOTE_BASE_PATH="${UPLOAD_PATH}" - if grep -qE 'B' <<< "$REPOSITORY"; then - if [[ "${TARGET}" == "main" ]]; then - REMOTE_PATH="debs-beta/" - else - REMOTE_PATH="debs-beta/extra/${{ matrix.release }}-${TARGET}/" - fi - elif grep -qE 'S' <<< "$REPOSITORY"; then - if [[ "${TARGET}" == "main" ]]; then - REMOTE_PATH="debs/" - else - REMOTE_PATH="debs/extra/${{ matrix.release }}-${TARGET}/" - fi - fi - - # Check if any .deb files for this package exist remotely with matching architecture - # Map Debian architecture names to common variants used in .deb filenames - case "${{ matrix.arch }}" in - amd64) - ARCH_PATTERNS=("amd64" "x86_64" "x64") - ;; - arm64) - ARCH_PATTERNS=("arm64" "aarch64") - ;; - armhf) - ARCH_PATTERNS=("armhf" "armv7h" "armv7l" "armv6") - ;; - riscv64) - ARCH_PATTERNS=("riscv64") - ;; - *) - ARCH_PATTERNS=("${{ matrix.arch }}") - ;; - esac - - # Build find command with architecture-specific patterns - FIND_CMD="find ${REMOTE_BASE_PATH}/${REMOTE_PATH} -type f -name '${PKG}_*.deb'" - for arch_pat in "${ARCH_PATTERNS[@]}"; do - FIND_CMD="$FIND_CMD -o -name '*_${arch_pat}.deb'" - done - FIND_CMD="$FIND_CMD 2>/dev/null" - - REMOTE_FILES=$(ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }} \ - ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }} \ - "$FIND_CMD" || true) - - if [[ -n "$REMOTE_FILES" ]]; then - echo "::notice::Found existing files on remote storage for $PKG (arch: ${{ matrix.arch }}):" - echo "$REMOTE_FILES" - # Get the version from the newest remote file - REMOTE_VERSION=$(echo "$REMOTE_FILES" | xargs -I {} basename {} | \ - sed "s/${PKG}_//" | sed "s/_.*//" | sort -V | tail -n 1) - echo "::notice::Remote version: $REMOTE_VERSION" - BEFORE_VERSION="$REMOTE_VERSION" - SKIP_DOWNLOAD=true - else - echo "::debug::No existing files found on remote storage for $PKG (arch: ${{ matrix.arch }}), will proceed with download" - fi - fi - - # Get version from main repository (only if not found on remote) - if [[ -z "$BEFORE_VERSION" ]]; then - BEFORE_VERSION="" - # Try main component, desktop component, then extra component + # Get version from main repository + BEFORE_VERSION="" + # Try main component, desktop component, then extra component for repo_component in "main" "${{ matrix.release }}-desktop" "${{ matrix.release }}-utils"; do # Build URL for Packages index CURRENT_PACKAGES_URL="${PACKAGES_URL}/dists/${{ matrix.release }}/${repo_component}/binary-${{ matrix.arch }}/Packages.gz" @@ -627,18 +560,8 @@ jobs: echo "::warning::Could not find version for $PKG in repository, assuming new package" BEFORE_VERSION="0" fi - fi echo "BEFORE_VERSION=${BEFORE_VERSION}" >> $GITHUB_OUTPUT - echo "SKIP_DOWNLOAD=${SKIP_DOWNLOAD}" >> $GITHUB_OUTPUT - - # Skip download if files already exist on remote and version check passed - if [[ "$SKIP_DOWNLOAD" == "true" ]]; then - echo "::notice::Skipping download - files already exist on remote storage with version $BEFORE_VERSION" - # Create a dummy file to satisfy subsequent steps - touch "${SOURCE}skip.txt" - else - echo "::notice::Proceeding with download..." # Download packages based on method if [[ ${METHOD} == gh ]]; then @@ -881,10 +804,9 @@ jobs: done < <(find "$root" -type f -name '*.deb' -printf '%h\0' | sort -zu) done fi - fi # End of SKIP_DOWNLOAD check # Repack deb files with Armbian version - if [[ "${{ matrix.name }}" == "firefox" ]] && [[ "$SKIP_DOWNLOAD" != "true" ]]; then + if [[ "${{ matrix.name }}" == "firefox" ]]; then apt-get install -y devscripts DEBS=($(sudo find "${SOURCE}"* -type f -name '*firefox_*.deb')) for d in "${DEBS[@]}"; do @@ -899,17 +821,11 @@ jobs: fi # Store info to GitHub Actions - if [[ "$SKIP_DOWNLOAD" == "true" ]]; then - # Use the remote version as AFTER_VERSION since we skipped download - AFTER_VERSION="$BEFORE_VERSION" - PKG_LINES="Skipped download - using existing files on remote storage" - else - AFTER_VERSION=$(find "$SOURCE" -type f -name "${INSTALL%% *}*.deb" -exec dpkg-deb -f {} Version \; | sort | uniq | tail -n 1 | cut -d":" -f2) - if [[ -z $AFTER_VERSION ]]; then - AFTER_VERSION=$(find "$SOURCE" -type f -name "*.deb" -exec dpkg-deb -f {} Version \; | sort | uniq | tail -n 1 | cut -d":" -f2) - fi - PKG_LINES="$(find "$SOURCE" -type f -name "*.deb" -printf "%f\n" 2>/dev/null | sort | sed 's/$/
/')" + AFTER_VERSION=$(find "$SOURCE" -type f -name "${INSTALL%% *}*.deb" -exec dpkg-deb -f {} Version \; | sort | uniq | tail -n 1 | cut -d":" -f2) + if [[ -z $AFTER_VERSION ]]; then + AFTER_VERSION=$(find "$SOURCE" -type f -name "*.deb" -exec dpkg-deb -f {} Version \; | sort | uniq | tail -n 1 | cut -d":" -f2) fi + PKG_LINES="$(find "$SOURCE" -type f -name "*.deb" -printf "%f\n" 2>/dev/null | sort | sed 's/$/
/')" echo "AFTER_VERSION=${AFTER_VERSION}" >> $GITHUB_OUTPUT # Determine if update is needed @@ -922,9 +838,6 @@ jobs: # Upload packages directly to remote storage to avoid race conditions # This prevents parallel jobs from overwriting each other's packages # Note: StrictHostKeychecking=no is used here; consider using proper known_hosts in production - - # Skip upload if download was skipped (files already exist on remote) - if [[ "$SKIP_DOWNLOAD" != "true" ]]; then if [[ ${TARGET} == main ]]; then # Upload to main repository directories if grep -qE 'B' <<< "$REPOSITORY"; then @@ -950,7 +863,6 @@ jobs: -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs/extra/${{ matrix.release }}-${TARGET}/ \; fi fi - fi # End of SKIP_DOWNLOAD upload check # Upload to repository if version changed if dpkg --compare-versions "$AFTER_VERSION" gt "$BEFORE_VERSION"; then @@ -961,12 +873,6 @@ jobs: echo "" >> $GITHUB_STEP_SUMMARY echo "packages:
$PKG_LINES" >> $GITHUB_STEP_SUMMARY - elif [[ "$SKIP_DOWNLOAD" == "true" ]]; then - # Add summary for skipped downloads - echo '| name | method | arch | release | needs_qemu | target | version | status |' >> $GITHUB_STEP_SUMMARY - echo '|--------|-------|------|---------|------------|--------|---------|--------|' >> $GITHUB_STEP_SUMMARY - echo "| ${{ matrix.name }} | ${{ matrix.method }} | ${{ matrix.arch }} | ${{ matrix.release }} | ${{ matrix.needs_qemu }} | ${{ matrix.target }} | $BEFORE_VERSION | ⏭️ Skipped (already on remote) |" >> $GITHUB_STEP_SUMMARY - elif [[ "${{ inputs.HIDE_NO_UPDATE }}" == "true" ]]; then # Exit if HIDE_NO_UPDATE is enabled and no update needed echo "::notice::No update needed for ${{ matrix.name }} on ${{ matrix.arch }}, exiting early" diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index 7e8b733c3..8d860ec11 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -158,6 +158,51 @@ jobs: rsync -av --omit-dir-times --no-perms --no-group ${INCOMING_PATH}/external/debs-beta ${STORAGE_PATH}/debs-beta 2>&1 | tee -a $GITHUB_STEP_SUMMARY || echo "Warning: Some files/attrs were not transferred (code 23)" >> $GITHUB_STEP_SUMMARY fi + prepare-repos: + name: "Prepare repositories" + needs: Copying + runs-on: repository + steps: + - name: Checkout build repository + uses: actions/checkout@v6 + with: + repository: armbian/build + path: build + clean: false + + - name: Checkout armbian.github.io repository + uses: actions/checkout@v6 + with: + repository: armbian/armbian.github.io + path: armbian-github-io + clean: false + + - name: Copy example.deb to repository subfolders + run: | + # Get all supported releases from build config + ALL_RELEASES=($(grep -rw build/config/distributions/*/support -ve 'eos' | cut -d"/" -f4)) + echo "Found ${#ALL_RELEASES[@]} releases: ${ALL_RELEASES[*]}" | tee -a $GITHUB_STEP_SUMMARY + + # Copy example.deb to all subfolders for debs + for RELEASE in "${ALL_RELEASES[@]}"; do + echo "Copying example.deb for debs/$RELEASE" | tee -a $GITHUB_STEP_SUMMARY + cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/$RELEASE/" + mkdir -p "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-utils" + mkdir -p "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-desktop" + cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-utils/" + cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-desktop/" + done + + # Copy example.deb to all subfolders for debs-beta + for RELEASE in "${ALL_RELEASES[@]}"; do + echo "Copying example.deb for debs-beta/$RELEASE" | tee -a $GITHUB_STEP_SUMMARY + cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/$RELEASE/" + mkdir -p "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-utils" + mkdir -p "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-desktop" + cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-utils/" + cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-desktop/" + done + fix-permissions: name: "Fix permissions" needs: external From fb1077e562fbcb28231e1aca3ed94e0ddc7f0f98 Mon Sep 17 00:00:00 2001 From: Igor Pecovnik Date: Sun, 11 Jan 2026 17:37:03 +0100 Subject: [PATCH 11/11] Fix upload logic --- .../infrastructure-download-external.yml | 37 +++++++++----- .../infrastructure-repository-update.yml | 51 +++++++++++++------ 2 files changed, 58 insertions(+), 30 deletions(-) diff --git a/.github/workflows/infrastructure-download-external.yml b/.github/workflows/infrastructure-download-external.yml index 5b741b007..809d4f307 100644 --- a/.github/workflows/infrastructure-download-external.yml +++ b/.github/workflows/infrastructure-download-external.yml @@ -325,6 +325,7 @@ jobs: ;; armhf|riscv64) runner='["ubuntu-latest"]' + runner='["docker", "X64"]' image_arch="amd64" # Use amd64 image with QEMU emulation ;; *) @@ -838,29 +839,37 @@ jobs: # Upload packages directly to remote storage to avoid race conditions # This prevents parallel jobs from overwriting each other's packages # Note: StrictHostKeychecking=no is used here; consider using proper known_hosts in production + # Always upload to debs-beta for external packages if [[ ${TARGET} == main ]]; then - # Upload to main repository directories - if grep -qE 'B' <<< "$REPOSITORY"; then - find "$SOURCE" -type f -name "*.deb" -exec \ - rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ - -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs-beta/ \; - fi - if grep -qE 'S' <<< "$REPOSITORY"; then + # Upload to main repository directory + find "$SOURCE" -type f -name "*.deb" -exec \ + rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ + -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs-beta/ \; + + # Upload to debs only if version is newer than what's in repository + # This is checked by comparing BEFORE_VERSION from the repository + if dpkg --compare-versions "$AFTER_VERSION" gt "$BEFORE_VERSION"; then + echo "::notice::Newer version $AFTER_VERSION found, uploading to debs" find "$SOURCE" -type f -name "*.deb" -exec \ rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs/ \; + else + echo "::notice::Version $AFTER_VERSION already exists in debs (has $BEFORE_VERSION), skipping upload" fi else - # Upload to specific release directories - if grep -qE 'B' <<< "$REPOSITORY"; then - find "$SOURCE" -type f -name "*.deb" -exec \ - rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ - -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs-beta/extra/${{ matrix.release }}-${TARGET}/ \; - fi - if grep -qE 'S' <<< "$REPOSITORY"; then + # Upload to specific release directory in debs-beta + find "$SOURCE" -type f -name "*.deb" -exec \ + rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ + -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs-beta/extra/${{ matrix.release }}-${TARGET}/ \; + + # Upload to debs only if version is newer than what's in repository + if dpkg --compare-versions "$AFTER_VERSION" gt "$BEFORE_VERSION"; then + echo "::notice::Newer version $AFTER_VERSION found, uploading to debs/extra" find "$SOURCE" -type f -name "*.deb" -exec \ rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs/extra/${{ matrix.release }}-${TARGET}/ \; + else + echo "::notice::Version $AFTER_VERSION already exists in debs (has $BEFORE_VERSION), skipping upload" fi fi diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index 8d860ec11..1eca0a1c5 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -22,9 +22,9 @@ env: CLEANUP_INPUT: true DRY_RUN_SYNC: false -#concurrency: -# group: pipeline -# cancel-in-progress: false +concurrency: + group: pipeline + cancel-in-progress: false jobs: @@ -43,7 +43,7 @@ jobs: external: name: "Download external" needs: Check - uses: armbian/armbian.github.io/.github/workflows/infrastructure-download-external.yml@always + uses: armbian/armbian.github.io/.github/workflows/infrastructure-download-external.yml@main with: ENABLED: ${{ inputs.download_external != false || github.event.client_payload.download_external != false }} SKIP_VERSION_CHECK: false @@ -153,9 +153,25 @@ jobs: esac # Always sync external - if [ -d "${INCOMING_PATH}/external" ]; then - rsync -av --omit-dir-times --no-perms --no-group ${INCOMING_PATH}/external/debs ${STORAGE_PATH}/debs 2>&1 | tee -a $GITHUB_STEP_SUMMARY || echo "Warning: Some files/attrs were not transferred (code 23)" >> $GITHUB_STEP_SUMMARY - rsync -av --omit-dir-times --no-perms --no-group ${INCOMING_PATH}/external/debs-beta ${STORAGE_PATH}/debs-beta 2>&1 | tee -a $GITHUB_STEP_SUMMARY || echo "Warning: Some files/attrs were not transferred (code 23)" >> $GITHUB_STEP_SUMMARY + if [ -d "${INCOMING_PATH}/external/debs" ]; then + rsync -av \ + --include='*/' \ + --include='*.deb' \ + --exclude='*' \ + --omit-dir-times --no-perms --no-group \ + "${INCOMING_PATH}/external/debs/" "${STORAGE_PATH}/debs/" \ + 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" || \ + echo "Warning: Some files/attrs were not transferred (code 23)" >> "$GITHUB_STEP_SUMMARY" + fi + if [ -d "${INCOMING_PATH}/external/debs-beta" ]; then + rsync -av \ + --include='*/' \ + --include='*.deb' \ + --exclude='*' \ + --omit-dir-times --no-perms --no-group \ + "${INCOMING_PATH}/external/debs-beta/" "${STORAGE_PATH}/debs-beta/" \ + 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" || \ + echo "Warning: Some files/attrs were not transferred (code 23)" >> "$GITHUB_STEP_SUMMARY" fi prepare-repos: @@ -179,33 +195,36 @@ jobs: - name: Copy example.deb to repository subfolders run: | + # Get all supported releases from build config ALL_RELEASES=($(grep -rw build/config/distributions/*/support -ve 'eos' | cut -d"/" -f4)) echo "Found ${#ALL_RELEASES[@]} releases: ${ALL_RELEASES[*]}" | tee -a $GITHUB_STEP_SUMMARY + + sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/$RELEASE/" + sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/$RELEASE/" + # Copy example.deb to all subfolders for debs for RELEASE in "${ALL_RELEASES[@]}"; do - echo "Copying example.deb for debs/$RELEASE" | tee -a $GITHUB_STEP_SUMMARY - cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/$RELEASE/" + echo "Copying example.deb for debs/$RELEASE and subfolders" | tee -a $GITHUB_STEP_SUMMARY mkdir -p "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-utils" mkdir -p "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-desktop" - cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-utils/" - cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-desktop/" + sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-utils/" + sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-desktop/" done # Copy example.deb to all subfolders for debs-beta for RELEASE in "${ALL_RELEASES[@]}"; do - echo "Copying example.deb for debs-beta/$RELEASE" | tee -a $GITHUB_STEP_SUMMARY - cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/$RELEASE/" + echo "Copying example.deb for debs-beta/$RELEASE and subfolders" | tee -a $GITHUB_STEP_SUMMARY mkdir -p "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-utils" mkdir -p "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-desktop" - cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-utils/" - cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-desktop/" + sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-utils/" + sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-desktop/" done fix-permissions: name: "Fix permissions" - needs: external + needs: prepare-repos runs-on: repository strategy: matrix: