From dca28c0b763a3a403f86084d470f55b5f45e143c Mon Sep 17 00:00:00 2001 From: Kishor Prins Date: Mon, 24 Nov 2025 09:57:21 -0500 Subject: [PATCH 1/2] Port back the build matrix to upstream --- .github/workflows/release.yml | 5 ++++- docker-bake.hcl | 39 ++++++++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 395d218e..29704cbc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,6 +35,9 @@ jobs: BUILDKIT_STEP_LOG_MAX_SIZE: 10485760 # This environment variable will override the VERSION variable in docker-bake.hcl VERSION: ${{ needs.prepare-release.outputs.version_tag }} # Use tag version (vX.Y.Z) for bake + strategy: + matrix: + build_target: ["cpu", "cpu-arm64", "gpu-arm64", "gpu"] steps: - name: Checkout repository uses: actions/checkout@v4 @@ -85,7 +88,7 @@ jobs: run: | echo "Building and pushing images for version ${{ needs.prepare-release.outputs.version_tag }}" # The VERSION env var above sets the tag for the bake file targets - docker buildx bake --push + docker buildx bake ${{ matrix.build_target }} --push create-release: needs: [prepare-release, build-images] diff --git a/docker-bake.hcl b/docker-bake.hcl index e29599a0..a66b70af 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -61,8 +61,41 @@ target "gpu" { } # Default group to build both CPU and GPU versions -group "default" { - targets = ["cpu", "gpu"] +target "cpu" { + inherits = ["_cpu_base"] + platforms = ["linux/amd64"] + tags = [ + "${REGISTRY}/${OWNER}/${REPO}-cpu:${VERSION}", + "${REGISTRY}/${OWNER}/${REPO}-cpu:latest" + ] +} + +target "cpu-arm64" { + inherits = ["_cpu_base"] + platforms = ["linux/arm64"] + tags = [ + "${REGISTRY}/${OWNER}/${REPO}-cpu:${VERSION}", + "${REGISTRY}/${OWNER}/${REPO}-cpu:latest" + ] +} + +# GPU target with multi-platform support +target "gpu" { + inherits = ["_gpu_base"] + platforms = ["linux/amd64"] + tags = [ + "${REGISTRY}/${OWNER}/${REPO}-gpu:${VERSION}", + "${REGISTRY}/${OWNER}/${REPO}-gpu:latest" + ] +} + +target "gpu-arm64" { + inherits = ["_gpu_base"] + platforms = ["linux/arm64"] + tags = [ + "${REGISTRY}/${OWNER}/${REPO}-gpu:${VERSION}", + "${REGISTRY}/${OWNER}/${REPO}-gpu:latest" + ] } # Development targets for faster local builds @@ -80,4 +113,4 @@ target "gpu-dev" { group "dev" { targets = ["cpu-dev", "gpu-dev"] -} \ No newline at end of file +} From 0ae1c33d0b679e6d0159867fbf89944174084ccf Mon Sep 17 00:00:00 2001 From: remsky Date: Sat, 13 Dec 2025 15:40:42 -0700 Subject: [PATCH 2/2] ci(builds): manifests, branch tagged builds, runners --- .github/workflows/release.yml | 129 ++++++++++++++++++++++++++-------- docker-bake.hcl | 40 +++++++---- 2 files changed, 129 insertions(+), 40 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 22e64261..405e39d5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,10 +3,16 @@ name: Create Release and Publish Docker Images on: push: branches: - - release # Trigger when commits are pushed to the release branch (e.g., after merging master) + - release # Auto-trigger only on release branch paths-ignore: - '**.md' - 'docs/**' + workflow_dispatch: # Manual trigger - explicitly specify branch + inputs: + branch_name: + description: 'Branch to build from (required)' + required: true + type: string jobs: prepare-release: @@ -22,25 +28,49 @@ jobs: id: get-version run: | VERSION_PLAIN=$(cat VERSION) - echo "version=${VERSION_PLAIN}" >> $GITHUB_OUTPUT - echo "version_tag=v${VERSION_PLAIN}" >> $GITHUB_OUTPUT # Add 'v' prefix for tag + + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + BRANCH_NAME="${{ inputs.branch_name }}" + else + BRANCH_NAME="${{ github.ref_name }}" + fi + + if [[ "$BRANCH_NAME" == "release" ]]; then + echo "version=${VERSION_PLAIN}" >> $GITHUB_OUTPUT + echo "version_tag=v${VERSION_PLAIN}" >> $GITHUB_OUTPUT + else + SAFE_BRANCH=$(echo "$BRANCH_NAME" | tr '/' '-' | tr '[:upper:]' '[:lower:]') + echo "version=${VERSION_PLAIN}-${SAFE_BRANCH}" >> $GITHUB_OUTPUT + echo "version_tag=v${VERSION_PLAIN}-${SAFE_BRANCH}" >> $GITHUB_OUTPUT + fi build-images: needs: prepare-release - runs-on: ubuntu-latest permissions: - packages: write # Needed to push images to GHCR + packages: write env: DOCKER_BUILDKIT: 1 BUILDKIT_STEP_LOG_MAX_SIZE: 10485760 - # These environment variables will override the defaults within docker-bake.hcl - VERSION: ${{ needs.prepare-release.outputs.version_tag }} # Use tag version (vX.Y.Z) for bake - strategy: - matrix: - build_target: ["cpu", "cpu-arm64", "gpu-arm64", "gpu"] - REGISTRY: ${{ vars.REGISTRY || 'ghcr.io' }} # Override with GitHub Action Environment Variable + VERSION: ${{ needs.prepare-release.outputs.version_tag }} + REGISTRY: ${{ vars.REGISTRY || 'ghcr.io' }} OWNER: ${{ vars.OWNER || 'remsky' }} REPO: ${{ vars.REPO || 'kokoro-fastapi' }} + strategy: + matrix: + include: + - build_target: "cpu" + platform: "linux/amd64" + runs_on: "ubuntu-latest" + - build_target: "gpu" + platform: "linux/amd64" + runs_on: "ubuntu-latest" + - build_target: "cpu" + platform: "linux/arm64" + runs_on: "ubuntu-24.04-arm" + - build_target: "gpu" + platform: "linux/arm64" + runs_on: "ubuntu-24.04-arm" + runs-on: ${{ matrix.runs_on }} steps: - name: Checkout repository uses: actions/checkout@v4 @@ -51,7 +81,6 @@ jobs: run: | TAG_NAME="${{ needs.prepare-release.outputs.version_tag }}" echo "Checking for existing tag: $TAG_NAME" - # Fetch tags explicitly just in case checkout didn't get them all git fetch --tags if git rev-parse "$TAG_NAME" >/dev/null 2>&1; then echo "::error::Tag $TAG_NAME already exists. Please increment the version in the VERSION file." @@ -60,7 +89,7 @@ jobs: echo "Tag $TAG_NAME does not exist. Proceeding with release." fi - - name: Free disk space # Optional: Keep as needed for large builds + - name: Free disk space run: | echo "Listing current disk space" df -h @@ -70,9 +99,6 @@ jobs: echo "Disk space after cleanup" df -h - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 # Use v3 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 # Use v3 with: @@ -87,30 +113,77 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push images using Docker Bake + - name: Build and push single-platform image run: | - echo "Building and pushing images for version ${{ needs.prepare-release.outputs.version_tag }}" - # The VERSION env var above sets the tag for the bake file targets - docker buildx bake ${{ matrix.build_target }} --push + PLATFORM="${{ matrix.platform }}" + BUILD_TARGET="${{ matrix.build_target }}" + VERSION_TAG="${{ needs.prepare-release.outputs.version_tag }}" + + echo "Building ${PLATFORM} image for ${BUILD_TARGET} version ${VERSION_TAG}" + + TARGET="${BUILD_TARGET}-$(echo ${PLATFORM} | cut -d'/' -f2)" + echo "Using bake target: $TARGET" + + docker buildx bake $TARGET --push --progress=plain - create-release: + create-manifests: needs: [prepare-release, build-images] runs-on: ubuntu-latest permissions: - contents: write # Needed to create releases + packages: write + env: + REGISTRY: ${{ vars.REGISTRY || 'ghcr.io' }} + OWNER: ${{ vars.OWNER || 'remsky' }} + REPO: ${{ vars.REPO || 'kokoro-fastapi' }} + strategy: + matrix: + build_target: ["cpu", "gpu"] + steps: + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create multi-platform manifest + run: | + VERSION_TAG="${{ needs.prepare-release.outputs.version_tag }}" + TARGET="${{ matrix.build_target }}" + REGISTRY="${{ env.REGISTRY }}" + OWNER="${{ env.OWNER }}" + REPO="${{ env.REPO }}" + + docker buildx imagetools create -t \ + ${REGISTRY}/${OWNER}/${REPO}-${TARGET}:${VERSION_TAG} \ + ${REGISTRY}/${OWNER}/${REPO}-${TARGET}:${VERSION_TAG}-amd64 \ + ${REGISTRY}/${OWNER}/${REPO}-${TARGET}:${VERSION_TAG}-arm64 + + if [[ "$VERSION_TAG" != *"-"* ]]; then + docker buildx imagetools create -t \ + ${REGISTRY}/${OWNER}/${REPO}-${TARGET}:latest \ + ${REGISTRY}/${OWNER}/${REPO}-${TARGET}:${VERSION_TAG}-amd64 \ + ${REGISTRY}/${OWNER}/${REPO}-${TARGET}:${VERSION_TAG}-arm64 + fi + + create-release: + needs: [prepare-release, create-manifests] + runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout repository uses: actions/checkout@v4 with: - fetch-depth: 0 # Fetch all history for release notes generation + fetch-depth: 0 - name: Create GitHub Release - uses: softprops/action-gh-release@v2 # Use v2 + uses: softprops/action-gh-release@v2 with: - tag_name: ${{ needs.prepare-release.outputs.version_tag }} # Use vX.Y.Z tag + tag_name: ${{ needs.prepare-release.outputs.version_tag }} name: Release ${{ needs.prepare-release.outputs.version_tag }} - generate_release_notes: true # Auto-generate release notes - draft: false # Publish immediately - prerelease: false # Mark as a stable release + generate_release_notes: true + draft: false + prerelease: false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/docker-bake.hcl b/docker-bake.hcl index a66b70af..8fd98bd6 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -60,13 +60,13 @@ target "gpu" { ] } -# Default group to build both CPU and GPU versions -target "cpu" { +# Individual platform targets for debugging/testing +target "cpu-amd64" { inherits = ["_cpu_base"] platforms = ["linux/amd64"] tags = [ - "${REGISTRY}/${OWNER}/${REPO}-cpu:${VERSION}", - "${REGISTRY}/${OWNER}/${REPO}-cpu:latest" + "${REGISTRY}/${OWNER}/${REPO}-cpu:${VERSION}-amd64", + "${REGISTRY}/${OWNER}/${REPO}-cpu:latest-amd64" ] } @@ -74,18 +74,17 @@ target "cpu-arm64" { inherits = ["_cpu_base"] platforms = ["linux/arm64"] tags = [ - "${REGISTRY}/${OWNER}/${REPO}-cpu:${VERSION}", - "${REGISTRY}/${OWNER}/${REPO}-cpu:latest" + "${REGISTRY}/${OWNER}/${REPO}-cpu:${VERSION}-arm64", + "${REGISTRY}/${OWNER}/${REPO}-cpu:latest-arm64" ] } -# GPU target with multi-platform support -target "gpu" { +target "gpu-amd64" { inherits = ["_gpu_base"] platforms = ["linux/amd64"] tags = [ - "${REGISTRY}/${OWNER}/${REPO}-gpu:${VERSION}", - "${REGISTRY}/${OWNER}/${REPO}-gpu:latest" + "${REGISTRY}/${OWNER}/${REPO}-gpu:${VERSION}-amd64", + "${REGISTRY}/${OWNER}/${REPO}-gpu:latest-amd64" ] } @@ -93,8 +92,8 @@ target "gpu-arm64" { inherits = ["_gpu_base"] platforms = ["linux/arm64"] tags = [ - "${REGISTRY}/${OWNER}/${REPO}-gpu:${VERSION}", - "${REGISTRY}/${OWNER}/${REPO}-gpu:latest" + "${REGISTRY}/${OWNER}/${REPO}-gpu:${VERSION}-arm64", + "${REGISTRY}/${OWNER}/${REPO}-gpu:latest-arm64" ] } @@ -114,3 +113,20 @@ target "gpu-dev" { group "dev" { targets = ["cpu-dev", "gpu-dev"] } + +# Build groups for different use cases +group "cpu-all" { + targets = ["cpu", "cpu-amd64", "cpu-arm64"] +} + +group "gpu-all" { + targets = ["gpu", "gpu-amd64", "gpu-arm64"] +} + +group "all" { + targets = ["cpu", "gpu"] +} + +group "individual-platforms" { + targets = ["cpu-amd64", "cpu-arm64", "gpu-amd64", "gpu-arm64"] +}