diff --git a/.github/workflows/infrastructure-download-external.yml b/.github/workflows/infrastructure-download-external.yml index 0973e1a64..809d4f307 100644 --- a/.github/workflows/infrastructure-download-external.yml +++ b/.github/workflows/infrastructure-download-external.yml @@ -65,6 +65,7 @@ env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.number }} PACKAGES_URL: "https://fi.mirror.armbian.de/apt" + UPLOAD_PATH: "storage/incoming/external" jobs: @@ -219,7 +220,7 @@ jobs: # Note: StrictHostKeychecking=no is used here; consider using proper known_hosts in production ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }} \ ${{ inputs.HOST_USER }}@${{ inputs.HOST_DEPLOY }} \ - "rm -rf storage/artifacts/*" + "rm -rf ${UPLOAD_PATH}/artifacts/*" - name: Checkout Armbian OS repository uses: actions/checkout@v6 @@ -309,27 +310,26 @@ jobs: # Force amd64 runner for aptly method due to stability issues on ARM if [[ "${METHOD:-aptly}" == "aptly" ]]; then # Always use amd64 runner for aptly, use QEMU for other arches - runner="ubuntu-latest" + runner='["docker", "X64"]' image_arch="amd64" else # Use architecture-specific runners for other methods (gh, direct) case "${arch}" in amd64) - runner="ubuntu-latest" - #runner="X64" + runner='["docker", "X64"]' image_arch="${arch}" ;; arm64) - runner="ubuntu-24.04-arm" - #runner="docker" + runner='["docker", "ARM64"]' image_arch="${arch}" ;; armhf|riscv64) - runner="ubuntu-latest" + runner='["ubuntu-latest"]' + runner='["docker", "X64"]' image_arch="amd64" # Use amd64 image with QEMU emulation ;; *) - runner="ubuntu-latest" + runner='["ubuntu-latest"]' image_arch="amd64" ;; esac @@ -347,7 +347,7 @@ jobs: --arg image_arch "${image_arch}" \ --arg registry "ghcr.io/${{ github.repository_owner }}" \ --argjson needs_qemu "${needs_qemu}" \ - '{name: $name, arch: $arch, release: $release, target: $target, method: $method, install: $install, runner: $runner, image_arch: $image_arch, registry: $registry, needs_qemu: $needs_qemu}' + '{name: $name, arch: $arch, release: $release, target: $target, method: $method, install: $install, runner: ($runner | fromjson), image_arch: $image_arch, registry: $registry, needs_qemu: $needs_qemu}' done done done | jq -s '{"include": .}' @@ -435,15 +435,20 @@ jobs: # Install appropriate keyring based on container type if grep -q "debian" /etc/os-release; then - apt-get -y install debian-keyring + apt-get -y install debian-keyring 2>&1 | grep -v "^gpg:" || true elif grep -q "ubuntu" /etc/os-release; then - apt-get -y install ubuntu-keyring + apt-get -y install ubuntu-keyring 2>&1 | grep -v "^gpg:" || true fi - # Import additional keys - gpg --no-default-keyring --keyring trustedkeys.gpg \ + # Import additional keys with timeout and suppress verbose output + # Use timeout to prevent hanging on unavailable keyservers + if timeout 30 gpg --no-default-keyring --keyring trustedkeys.gpg \ --keyserver keyserver.ubuntu.com \ - --recv-keys 648ACFD622F3D138 0E98404D386FA1D9 + --recv-keys 648ACFD622F3D138 0E98404D386FA1D9 2>&1 | grep -v "not changed\|unchanged\|not checked" || true; then + echo "::notice::GPG keys imported successfully" + else + echo "::warning::Some GPG keys could not be imported from keyserver (non-critical)" + fi - name: "Prepare machine" id: preparing @@ -460,11 +465,18 @@ jobs: [[ "${SIGNATURES}" == "ignore" ]] && APTLY_CONF+="-ignore-signatures " # Read existing releases and create folder structure + rm -rf build/output/{debs,debs-beta} ALL_RELEASES=($(grep -rw build/config/distributions/*/support -ve 'eos' | cut -d"/" -f4 )) for i in ${ALL_RELEASES[@]}; do mkdir -p build/output/{debs,debs-beta}/${i} mkdir -p build/output/{debs,debs-beta}/extra/${i}-utils mkdir -p build/output/{debs,debs-beta}/extra/${i}-desktop + # Add example.deb to each subfolder to ensure empty repos are published + for repo_root in build/output/debs build/output/debs-beta; do + cp build/tools/repository/example.deb ${repo_root}/${i}/ + cp build/tools/repository/example.deb ${repo_root}/extra/${i}-utils/ + cp build/tools/repository/example.deb ${repo_root}/extra/${i}-desktop/ + done done needs_qemu='${{ matrix.needs_qemu }}' @@ -507,6 +519,7 @@ jobs: . os/external/${{ matrix.name }}.conf SOURCE="temp/" + rm -rf ${SOURCE} mkdir -p ${SOURCE} # Get current version from Armbian repository @@ -521,13 +534,7 @@ jobs: # Get version from main repository BEFORE_VERSION="" - - # Skip version check if SKIP_VERSION_CHECK is enabled (for force repopulation) - if [[ "${{ inputs.SKIP_VERSION_CHECK }}" == "true" ]]; then - echo "::notice::SKIP_VERSION_CHECK enabled - forcing upload regardless of existing versions" - BEFORE_VERSION="0" - else - # Try main component, desktop component, then extra component + # Try main component, desktop component, then extra component for repo_component in "main" "${{ matrix.release }}-desktop" "${{ matrix.release }}-utils"; do # Build URL for Packages index CURRENT_PACKAGES_URL="${PACKAGES_URL}/dists/${{ matrix.release }}/${repo_component}/binary-${{ matrix.arch }}/Packages.gz" @@ -554,10 +561,9 @@ jobs: echo "::warning::Could not find version for $PKG in repository, assuming new package" BEFORE_VERSION="0" fi - fi echo "BEFORE_VERSION=${BEFORE_VERSION}" >> $GITHUB_OUTPUT - + # Download packages based on method if [[ ${METHOD} == gh ]]; then # GitHub release download method @@ -833,30 +839,37 @@ jobs: # Upload packages directly to remote storage to avoid race conditions # This prevents parallel jobs from overwriting each other's packages # Note: StrictHostKeychecking=no is used here; consider using proper known_hosts in production - + # Always upload to debs-beta for external packages if [[ ${TARGET} == main ]]; then - # Upload to main repository directories - if grep -qE 'B' <<< "$REPOSITORY"; then + # Upload to main repository directory + find "$SOURCE" -type f -name "*.deb" -exec \ + rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ + -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs-beta/ \; + + # Upload to debs only if version is newer than what's in repository + # This is checked by comparing BEFORE_VERSION from the repository + if dpkg --compare-versions "$AFTER_VERSION" gt "$BEFORE_VERSION"; then + echo "::notice::Newer version $AFTER_VERSION found, uploading to debs" find "$SOURCE" -type f -name "*.deb" -exec \ rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ - -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:storage/debs-beta/ \; - fi - if grep -qE 'S' <<< "$REPOSITORY"; then - find "$SOURCE" -type f -name "*.deb" -exec \ - rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ - -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:storage/debs/ \; + -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs/ \; + else + echo "::notice::Version $AFTER_VERSION already exists in debs (has $BEFORE_VERSION), skipping upload" fi else - # Upload to specific release directories - if grep -qE 'B' <<< "$REPOSITORY"; then + # Upload to specific release directory in debs-beta + find "$SOURCE" -type f -name "*.deb" -exec \ + rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ + -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs-beta/extra/${{ matrix.release }}-${TARGET}/ \; + + # Upload to debs only if version is newer than what's in repository + if dpkg --compare-versions "$AFTER_VERSION" gt "$BEFORE_VERSION"; then + echo "::notice::Newer version $AFTER_VERSION found, uploading to debs/extra" find "$SOURCE" -type f -name "*.deb" -exec \ rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ - -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:storage/debs-beta/extra/${{ matrix.release }}-${TARGET}/ \; - fi - if grep -qE 'S' <<< "$REPOSITORY"; then - find "$SOURCE" -type f -name "*.deb" -exec \ - rsync -e "ssh -o StrictHostKeychecking=no -p ${{ secrets.HOST_UPLOAD_PORT }}" \ - -arvc --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:storage/debs/extra/${{ matrix.release }}-${TARGET}/ \; + -arvc --mkpath --ignore-existing {} ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:${UPLOAD_PATH}/debs/extra/${{ matrix.release }}-${TARGET}/ \; + else + echo "::notice::Version $AFTER_VERSION already exists in debs (has $BEFORE_VERSION), skipping upload" fi fi diff --git a/.github/workflows/infrastructure-repository-update.yml b/.github/workflows/infrastructure-repository-update.yml index 97bb293fe..1eca0a1c5 100644 --- a/.github/workflows/infrastructure-repository-update.yml +++ b/.github/workflows/infrastructure-repository-update.yml @@ -40,8 +40,29 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TEAM: "Release manager" - Copying: + external: + name: "Download external" needs: Check + uses: armbian/armbian.github.io/.github/workflows/infrastructure-download-external.yml@main + with: + ENABLED: ${{ inputs.download_external != false || github.event.client_payload.download_external != false }} + SKIP_VERSION_CHECK: false + ACCESS_NAME: armbian + BUILD_RUNNER: "docker" + HOST_DEPLOY: "repo.armbian.com" + PURGE: ${{ inputs.purge_external || false }} + secrets: + GPG_KEY1: ${{ secrets.GPG_KEY3 }} + GPG_KEY2: ${{ secrets.GPG_KEY4 }} + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + KEY_UPLOAD: ${{ secrets.KEY_UPLOAD }} + HOST_UPLOAD: ${{ secrets.HOST_UPLOAD }} + HOST_UPLOAD_USER: ${{ secrets.HOST_UPLOAD_USER }} + HOST_UPLOAD_PORT: ${{ secrets.HOST_UPLOAD_PORT }} + KNOWN_HOSTS_ARMBIAN_UPLOAD: ${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }} + + Copying: + needs: external runs-on: repository steps: @@ -131,31 +152,79 @@ jobs: ;; esac - external: - name: "Download external" + # Always sync external + if [ -d "${INCOMING_PATH}/external/debs" ]; then + rsync -av \ + --include='*/' \ + --include='*.deb' \ + --exclude='*' \ + --omit-dir-times --no-perms --no-group \ + "${INCOMING_PATH}/external/debs/" "${STORAGE_PATH}/debs/" \ + 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" || \ + echo "Warning: Some files/attrs were not transferred (code 23)" >> "$GITHUB_STEP_SUMMARY" + fi + if [ -d "${INCOMING_PATH}/external/debs-beta" ]; then + rsync -av \ + --include='*/' \ + --include='*.deb' \ + --exclude='*' \ + --omit-dir-times --no-perms --no-group \ + "${INCOMING_PATH}/external/debs-beta/" "${STORAGE_PATH}/debs-beta/" \ + 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" || \ + echo "Warning: Some files/attrs were not transferred (code 23)" >> "$GITHUB_STEP_SUMMARY" + fi + + prepare-repos: + name: "Prepare repositories" needs: Copying - uses: armbian/armbian.github.io/.github/workflows/infrastructure-download-external.yml@main - with: - ENABLED: ${{ inputs.download_external != false || github.event.client_payload.download_external != false }} - SKIP_VERSION_CHECK: false - ACCESS_NAME: armbian - BUILD_RUNNER: "ubuntu-latest" - HOST_DEPLOY: "repo.armbian.com" - PURGE: ${{ inputs.purge_external || false }} - secrets: - GPG_KEY1: ${{ secrets.GPG_KEY3 }} - GPG_KEY2: ${{ secrets.GPG_KEY4 }} - ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} - KEY_UPLOAD: ${{ secrets.KEY_UPLOAD }} - HOST_UPLOAD: ${{ secrets.HOST_UPLOAD }} - HOST_UPLOAD_USER: ${{ secrets.HOST_UPLOAD_USER }} - HOST_UPLOAD_PORT: ${{ secrets.HOST_UPLOAD_PORT }} - KNOWN_HOSTS_ARMBIAN_UPLOAD: ${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }} + runs-on: repository + steps: + - name: Checkout build repository + uses: actions/checkout@v6 + with: + repository: armbian/build + path: build + clean: false + + - name: Checkout armbian.github.io repository + uses: actions/checkout@v6 + with: + repository: armbian/armbian.github.io + path: armbian-github-io + clean: false + + - name: Copy example.deb to repository subfolders + run: | + + # Get all supported releases from build config + ALL_RELEASES=($(grep -rw build/config/distributions/*/support -ve 'eos' | cut -d"/" -f4)) + echo "Found ${#ALL_RELEASES[@]} releases: ${ALL_RELEASES[*]}" | tee -a $GITHUB_STEP_SUMMARY + + + sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/$RELEASE/" + sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/$RELEASE/" + # Copy example.deb to all subfolders for debs + for RELEASE in "${ALL_RELEASES[@]}"; do + echo "Copying example.deb for debs/$RELEASE and subfolders" | tee -a $GITHUB_STEP_SUMMARY + mkdir -p "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-utils" + mkdir -p "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-desktop" + sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-utils/" + sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs/extra/$RELEASE-desktop/" + done + + # Copy example.deb to all subfolders for debs-beta + for RELEASE in "${ALL_RELEASES[@]}"; do + echo "Copying example.deb for debs-beta/$RELEASE and subfolders" | tee -a $GITHUB_STEP_SUMMARY + mkdir -p "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-utils" + mkdir -p "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-desktop" + sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-utils/" + sudo cp build/tools/repository/example.deb "${{ env.STORAGE_PATH }}/debs-beta/extra/$RELEASE-desktop/" + done fix-permissions: name: "Fix permissions" - needs: external + needs: prepare-repos runs-on: repository strategy: matrix: @@ -333,7 +402,7 @@ jobs: with: repository: armbian/build fetch-depth: 1 - ref: main + ref: always clean: false - name: "Build repository ${{ matrix.repository.name }}" @@ -412,7 +481,7 @@ jobs: with: repository: armbian/build clean: false - ref: main + ref: always fetch-depth: 1 path: build @@ -534,7 +603,7 @@ jobs: uses: actions/checkout@v6 with: repository: armbian/build - ref: main + ref: always fetch-depth: 1 clean: false @@ -646,7 +715,7 @@ jobs: uses: actions/checkout@v6 with: repository: armbian/build - ref: main + ref: always fetch-depth: 1 clean: false