diff --git a/.github/workflows/flyte-binary-v2.yml b/.github/workflows/flyte-binary-v2.yml index 23d2c70d9b..006c064080 100644 --- a/.github/workflows/flyte-binary-v2.yml +++ b/.github/workflows/flyte-binary-v2.yml @@ -100,11 +100,11 @@ jobs: path: docker/demo-bundled/images/tar - name: Login to GitHub Container Registry if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - uses: docker/login-action@v2 + uses: docker/login-action@v4 with: registry: ghcr.io - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" + username: ${{ secrets.FLYTE_BOT_USERNAME }} + password: ${{ secrets.FLYTE_BOT_PAT }} - name: Build and push Image if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} uses: docker/build-push-action@v6 @@ -153,11 +153,19 @@ jobs: type=sha,format=long, - name: Login to GitHub Container Registry if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} - uses: docker/login-action@v2 + uses: docker/login-action@v4 with: registry: ghcr.io - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" + username: ${{ secrets.FLYTE_BOT_USERNAME }} + password: ${{ secrets.FLYTE_BOT_PAT }} + - name: Save console image tarballs + if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + run: | + mkdir -p docker/demo-bundled/images/tar/amd64 docker/demo-bundled/images/tar/arm64 + docker pull --platform linux/amd64 ghcr.io/flyteorg/flyte-client-v2:latest + docker save ghcr.io/flyteorg/flyte-client-v2:latest -o docker/demo-bundled/images/tar/amd64/flyte-client-v2.tar + docker pull --platform linux/arm64 ghcr.io/flyteorg/flyte-client-v2:latest + docker save ghcr.io/flyteorg/flyte-client-v2:latest -o docker/demo-bundled/images/tar/arm64/flyte-client-v2.tar - name: Build and push multi-arch image uses: docker/build-push-action@v6 with: diff --git a/.github/workflows/flyte-core-v2.yaml b/.github/workflows/flyte-core-v2.yaml new file mode 100644 index 0000000000..3d534a65cc --- /dev/null +++ b/.github/workflows/flyte-core-v2.yaml @@ -0,0 +1,114 @@ +name: Build & Push Flyte Core v2 Images + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +on: + push: + branches: + - v2 + paths: + - "docker/Dockerfile.*" + - "docker/docker-bake.hcl" + - "actions/**" + - "cache_service/**" + - "dataproxy/**" + - "events/**" + - "executor/**" + - "runs/**" + - "flytestdlib/**" + - "flyteplugins/**" + - "flyteidl2/**" + - "gen/**" + - "go.mod" + - "go.sum" + pull_request: + branches: + - v2 + paths: + - "docker/Dockerfile.*" + - "docker/docker-bake.hcl" + - "actions/**" + - "cache_service/**" + - "dataproxy/**" + - "events/**" + - "executor/**" + - "runs/**" + - "flytestdlib/**" + - "flyteplugins/**" + - "flyteidl2/**" + - "gen/**" + - "go.mod" + - "go.sum" + +jobs: + prepare: + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.generate.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + sparse-checkout: docker/docker-bake.hcl + sparse-checkout-cone-mode: false + - name: Generate matrix + id: generate + uses: docker/bake-action/subaction/matrix@v7 + with: + files: docker/docker-bake.hcl + target: default + fields: platforms + build-and-push: + needs: prepare + runs-on: ${{ startsWith(matrix.platforms, 'linux/arm') && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare.outputs.matrix) }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Login to GitHub Container Registry + if: github.event_name == 'push' + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ secrets.FLYTE_BOT_USERNAME }} + password: ${{ secrets.FLYTE_BOT_PAT }} + + - name: Set short Git SHA as tag + run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + + - name: Build multi-arch image (PR) + if: github.event_name == 'pull_request' + uses: docker/bake-action@v7 + with: + source: . + files: docker/docker-bake.hcl + targets: ${{ matrix.target }} + set: | + *.platform=${{ matrix.platforms }} + *.args.tags="ghcr.io/flyteorg/flyte/${{ matrix.target }}:${{ env.SHORT_SHA }}" + + + - name: Build and push images + if: github.event_name == 'push' + uses: docker/bake-action@v7 + with: + source: . + files: docker/docker-bake.hcl + push: true + targets: ${{ matrix.target }} + set: | + *.platform=${{ matrix.platforms }} + *.args.tags="ghcr.io/flyteorg/flyte/${{ matrix.target }}:${{ env.SHORT_SHA }}" + \ No newline at end of file diff --git a/.github/workflows/git_town.yml b/.github/workflows/git_town.yml index 478ac9bfdb..e7c1695a2f 100644 --- a/.github/workflows/git_town.yml +++ b/.github/workflows/git_town.yml @@ -1,7 +1,7 @@ name: Git Town on: - pull_request: + pull_request_target: branches: - '**' diff --git a/actions/config.yaml b/actions/config.yaml new file mode 100644 index 0000000000..6d922789a9 --- /dev/null +++ b/actions/config.yaml @@ -0,0 +1,2 @@ +# Actions service configuration +actions: {} diff --git a/docker/Dockerfile.actions b/docker/Dockerfile.actions new file mode 100644 index 0000000000..4daa6085d8 --- /dev/null +++ b/docker/Dockerfile.actions @@ -0,0 +1,34 @@ +# syntax=docker/dockerfile:1 + +# Build stage +ARG GO_VERSION +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /src +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +COPY actions/ actions/ +COPY flytestdlib/ flytestdlib/ +COPY flyteplugins/ flyteplugins/ +COPY flyteidl2/ flyteidl2/ +COPY gen/ gen/ +COPY executor/ executor/ + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -trimpath -o actions-service actions/cmd/main.go + +# Final stage +FROM gcr.io/distroless/static-debian12 + +COPY --from=builder /src/actions-service / +# Copy default config +COPY --from=builder /src/actions/config.yaml / + +ENTRYPOINT ["/actions-service"] diff --git a/docker/Dockerfile.cache_service b/docker/Dockerfile.cache_service new file mode 100644 index 0000000000..9270221cbb --- /dev/null +++ b/docker/Dockerfile.cache_service @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1 + +# Build stage +ARG GO_VERSION +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /src +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +COPY cache_service/ cache_service/ +COPY flytestdlib/ flytestdlib/ +COPY gen/ gen/ + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -trimpath -o cache-service cache_service/cmd/main.go + +# Final stage +FROM gcr.io/distroless/static-debian12 + +COPY --from=builder /src/cache-service / + +ENTRYPOINT ["/cache-service"] diff --git a/docker/Dockerfile.dataproxy b/docker/Dockerfile.dataproxy new file mode 100644 index 0000000000..c707ccd7f9 --- /dev/null +++ b/docker/Dockerfile.dataproxy @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1 + +# Build stage +ARG GO_VERSION +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /src +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +COPY dataproxy/ dataproxy/ +COPY flytestdlib/ flytestdlib/ +COPY gen/ gen/ + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -trimpath -o dataproxy dataproxy/cmd/main.go + +# Final stage +FROM gcr.io/distroless/static-debian12 + +COPY --from=builder /src/dataproxy / + +ENTRYPOINT ["/dataproxy"] diff --git a/docker/Dockerfile.events b/docker/Dockerfile.events new file mode 100644 index 0000000000..2e943bc192 --- /dev/null +++ b/docker/Dockerfile.events @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1 + +# Build stage +ARG GO_VERSION +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /src +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +COPY events/ events/ +COPY flytestdlib/ flytestdlib/ +COPY gen/ gen/ + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -trimpath -o events events/cmd/main.go + +# Final stage +FROM gcr.io/distroless/static-debian12 + +COPY --from=builder /src/events / + +ENTRYPOINT ["/events"] diff --git a/docker/Dockerfile.executor b/docker/Dockerfile.executor new file mode 100644 index 0000000000..7bbae24b6f --- /dev/null +++ b/docker/Dockerfile.executor @@ -0,0 +1,31 @@ +# syntax=docker/dockerfile:1 + +# Build stage +ARG GO_VERSION +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /src +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +COPY executor/ executor/ +COPY flyteplugins/ flyteplugins/ +COPY flytestdlib/ flytestdlib/ +COPY flyteidl2/ flyteidl2/ +COPY gen/ gen/ + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -trimpath -o executor executor/cmd/main.go + +# Final stage +FROM gcr.io/distroless/static-debian12 + +COPY --from=builder /src/executor / + +ENTRYPOINT ["/executor"] diff --git a/docker/Dockerfile.runs b/docker/Dockerfile.runs new file mode 100644 index 0000000000..43b4aafd87 --- /dev/null +++ b/docker/Dockerfile.runs @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile:1 + +# Build stage +ARG GO_VERSION +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS builder + +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /src +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +COPY runs/ runs/ +COPY flytestdlib/ flytestdlib/ +COPY gen/ gen/ + +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} \ + go build -trimpath -o runs-service runs/cmd/main.go + +# Final stage +FROM gcr.io/distroless/static-debian12 + +COPY --from=builder /src/runs-service / +COPY --from=builder /src/runs/config.yaml / + +ENTRYPOINT ["/runs-service"] diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl new file mode 100644 index 0000000000..f816f9d084 --- /dev/null +++ b/docker/docker-bake.hcl @@ -0,0 +1,66 @@ +variable "IMAGE_REPO_PREFIX" { + default = "ghcr.io/flyteorg/flyte" +} + +variable "IMAGE_SHA_SHORT_TAG" { + default = "latest" +} + +variable "GO_VERSION" { + default = "1.24" +} + +group "default" { + targets = [ + "actions", + "cache_service", + "dataproxy", + "events", + "executor", + "runs", + ] +} + +target "_common" { + context = "." + platforms = ["linux/arm64", "linux/amd64"] + args = { + GO_VERSION = "${GO_VERSION}" + } +} + +target "actions" { + inherits = ["_common"] + dockerfile = "docker/Dockerfile.actions" + tags = ["${IMAGE_REPO_PREFIX}/actions:${IMAGE_SHA_SHORT_TAG}"] +} + +target "cache_service" { + inherits = ["_common"] + dockerfile = "docker/Dockerfile.cache_service" + tags = ["${IMAGE_REPO_PREFIX}/cache_service:${IMAGE_SHA_SHORT_TAG}"] +} + +target "dataproxy" { + inherits = ["_common"] + dockerfile = "docker/Dockerfile.dataproxy" + tags = ["${IMAGE_REPO_PREFIX}/dataproxy:${IMAGE_SHA_SHORT_TAG}"] +} + +target "events" { + inherits = ["_common"] + dockerfile = "docker/Dockerfile.events" + tags = ["${IMAGE_REPO_PREFIX}/events:${IMAGE_SHA_SHORT_TAG}"] +} + +target "executor" { + inherits = ["_common"] + dockerfile = "docker/Dockerfile.executor" + tags = ["${IMAGE_REPO_PREFIX}/executor:${IMAGE_SHA_SHORT_TAG}"] +} + +target "runs" { + inherits = ["_common"] + dockerfile = "docker/Dockerfile.runs" + tags = ["${IMAGE_REPO_PREFIX}/runs:${IMAGE_SHA_SHORT_TAG}"] +}