diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000000..5414d4b45fb
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,39 @@
+# Git
+.git
+.gitignore
+.gitattributes
+
+# CI/CD
+.github
+
+# Generated files
+gen/python/
+gen/rust/
+gen/ts/
+
+# Dependencies
+node_modules/
+target/
+**/Cargo.lock
+*.pyc
+__pycache__/
+.kube/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log
+
+# Test coverage
+coverage/
+*.coverage
+.pytest_cache/
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 00000000000..2fe5308674d
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,5 @@
+# Mark generated protobuf code so GitHub hides it in diffs by default
+gen/** linguist-generated=true
+
+# Mark generated mocks so GitHub hides them in diffs by default
+**/mocks/mocks.go linguist-generated=true
\ No newline at end of file
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 00000000000..c50f524f2b3
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,71 @@
+## Tracking issue
+
+
+## Why are the changes needed?
+
+
+
+## What changes were proposed in this pull request?
+
+
+
+## How was this patch tested?
+
+
+### Labels
+
+Please add one or more of the following labels to categorize your PR:
+- **added**: For new features.
+- **changed**: For changes in existing functionality.
+- **deprecated**: For soon-to-be-removed features.
+- **removed**: For features being removed.
+- **fixed**: For any bug fixed.
+- **security**: In case of vulnerabilities
+
+This is important to improve the readability of release notes.
+
+### Setup process
+
+### Screenshots
+
+## Check all the applicable boxes
+
+- [ ] I updated the documentation accordingly.
+- [ ] All new and existing tests passed.
+- [ ] All commits are signed-off.
+
+## Related PRs
+
+
+
+## Stack
+
+If you do use [`git town`](https://www.git-town.com/introduction) to manage PR Stacks, the stack relevant to this PR
+will show below. Otherwise, you can ignore this section.
+
+
+
+## Docs link
+
+
diff --git a/.github/actions/setup-python-env/action.yml b/.github/actions/setup-python-env/action.yml
new file mode 100644
index 00000000000..cab3a17b968
--- /dev/null
+++ b/.github/actions/setup-python-env/action.yml
@@ -0,0 +1,47 @@
+name: "Setup Python Environment"
+description: "Set up Python environment for the given Python version"
+
+inputs:
+ python-version:
+ description: "Python version to use"
+ required: true
+ default: "3.12"
+ uv-version:
+ description: "uv version to use"
+ required: true
+ default: "0.8.4"
+ working-directory:
+ description: "Default working directory for all steps"
+ required: false
+ default: ""
+
+runs:
+ using: "composite"
+ steps:
+ - uses: actions/setup-python@v5
+ with:
+ python-version: ${{ inputs.python-version }}
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@v6
+ id: setup-uv
+ with:
+ version: ${{ inputs.uv-version }}
+ enable-cache: 'true'
+ cache-suffix: ${{ inputs.python-version }}
+ cache-dependency-glob: |
+ pyproject.toml
+ uv.lock
+ working-directory: ${{ inputs.working-directory }}
+
+ - name: Install Python dependencies
+ if: steps.setup-uv.outputs.cache-hit == 'false'
+ working-directory: ${{ inputs.working-directory }}
+ run: uv sync --all-groups --frozen
+ shell: bash
+
+ - name: List Python dependencies
+ if: steps.setup-uv.outputs.cache-hit == 'true'
+ working-directory: ${{ inputs.working-directory }}
+ run: uv pip list
+ shell: bash
diff --git a/.github/workflows/build-ci-image.yml b/.github/workflows/build-ci-image.yml
new file mode 100644
index 00000000000..dd90db43e42
--- /dev/null
+++ b/.github/workflows/build-ci-image.yml
@@ -0,0 +1,158 @@
+name: Build and Publish CI Docker Image
+
+on:
+ push:
+ branches:
+ - main
+ - v2
+ paths:
+ - 'gen.Dockerfile'
+ - '.github/workflows/build-ci-image.yml'
+ pull_request:
+ paths:
+ - 'gen.Dockerfile'
+ - '.github/workflows/build-ci-image.yml'
+ workflow_dispatch:
+ inputs:
+ force_rebuild:
+ description: 'Force rebuild of the image'
+ required: false
+ default: 'false'
+
+env:
+ REGISTRY: ghcr.io
+ IMAGE_NAME: ${{ github.repository }}/ci
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ packages: write
+ pull-requests: write
+
+ outputs:
+ image-tag: ${{ steps.set-tag.outputs.tag }}
+
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Log in to GitHub Container Registry
+ uses: docker/login-action@v3
+ with:
+ registry: ${{ env.REGISTRY }}
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Extract metadata (tags, labels)
+ id: meta
+ uses: docker/metadata-action@v5
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+ tags: |
+ # Use branch name for branch pushes
+ type=ref,event=branch
+ # Use PR number for pull requests
+ type=ref,event=pr
+ # Use 'latest' tag for main branch
+ type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+ # Use 'v2' tag for v2 branch
+ type=raw,value=v2,enable=${{ github.ref == 'refs/heads/v2' }}
+ # Add git sha as tag
+ type=sha,prefix=${{ github.head_ref }}-,enable=${{ github.ref != 'refs/heads/v2' && github.ref != 'refs/heads/main' }}
+
+ - name: Set image tag output
+ id: set-tag
+ run: |
+ if [ "${{ github.event_name }}" == "pull_request" ]; then
+ echo "tag=pr-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
+ elif [ "${{ github.ref }}" == "refs/heads/v2" ]; then
+ echo "tag=v2" >> $GITHUB_OUTPUT
+ elif [ "${{ github.ref }}" == "refs/heads/main" ]; then
+ echo "tag=latest" >> $GITHUB_OUTPUT
+ else
+ echo "tag=$(echo ${{ github.ref }} | sed 's/refs\/heads\///')" >> $GITHUB_OUTPUT
+ fi
+
+ - name: Build and push Docker image
+ uses: docker/build-push-action@v6
+ with:
+ context: .
+ file: ./gen.Dockerfile
+ push: true
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+ platforms: linux/amd64,linux/arm64
+ # Multi-layer caching strategy for speed
+ cache-from: |
+ type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
+ type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.set-tag.outputs.tag }}
+ type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:v2
+ type=gha
+ cache-to: |
+ type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
+ type=gha,mode=max
+ # Enable BuildKit features for better caching
+ build-args: |
+ BUILDKIT_INLINE_CACHE=1
+
+ - name: Image digest
+ run: echo "Image built with digest ${{ steps.meta.outputs.digest }}"
+
+ - name: Comment on PR with image tag
+ if: github.event_name == 'pull_request'
+ uses: actions/github-script@v7
+ with:
+ script: |
+ const tag = '${{ steps.set-tag.outputs.tag }}';
+ const registry = '${{ env.REGISTRY }}';
+ const imageName = '${{ env.IMAGE_NAME }}';
+ const fullImage = `${registry}/${imageName}:${tag}`;
+
+ const comment = `## 🐳 Docker CI Image Built
+
+ The CI Docker image has been built and pushed for this PR!
+
+ **Image:** \`${fullImage}\`
+
+ This image will be automatically used by CI workflows in this PR.
+
+ To test locally:
+ \`\`\`bash
+ make gen DOCKER_CI_IMAGE=${fullImage}
+ \`\`\`
+ `;
+
+ // Find existing comment
+ const { data: comments } = await github.rest.issues.listComments({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ });
+
+ const botComment = comments.find(comment =>
+ comment.user.type === 'Bot' &&
+ comment.body.includes('🐳 Docker CI Image Built')
+ );
+
+ if (botComment) {
+ // Update existing comment
+ await github.rest.issues.updateComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ comment_id: botComment.id,
+ body: comment
+ });
+ } else {
+ // Create new comment
+ await github.rest.issues.createComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ body: comment
+ });
+ }
diff --git a/.github/workflows/check-generate.yml b/.github/workflows/check-generate.yml
new file mode 100644
index 00000000000..af78f2297af
--- /dev/null
+++ b/.github/workflows/check-generate.yml
@@ -0,0 +1,167 @@
+name: Check Generated Files
+
+on:
+ pull_request:
+ types: [opened, synchronize, reopened]
+
+jobs:
+ setup:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ actions: read
+ packages: read
+ outputs:
+ image: ${{ steps.docker-image.outputs.image }}
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Determine Docker image
+ id: docker-image
+ run: |
+ # Check if gen.Dockerfile or build workflow was modified
+ git fetch --depth=1 origin ${{ github.base_ref }}
+ if git diff --name-only FETCH_HEAD..HEAD | grep -E '^(gen\.Dockerfile|Dockerfile\.ci|\.github/workflows/build-ci-image\.yml)$'; then
+ echo "modified=true" >> $GITHUB_OUTPUT
+ echo "image=ghcr.io/flyteorg/flyte/ci:pr-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
+ echo "📦 gen.Dockerfile modified - will use PR-specific image"
+ else
+ echo "modified=false" >> $GITHUB_OUTPUT
+ echo "image=ghcr.io/flyteorg/flyte/ci:v2" >> $GITHUB_OUTPUT
+ echo "📦 Using default v2 image"
+ fi
+
+ - name: Wait for Docker image build workflow
+ if: steps.docker-image.outputs.modified == 'true'
+ uses: actions/github-script@v7
+ with:
+ script: |
+ const maxAttempts = 60; // 20 minutes max
+ const delaySeconds = 20;
+
+ console.log('⏳ Waiting for Docker image build workflow to complete...');
+
+ for (let attempt = 0; attempt < maxAttempts; attempt++) {
+ // Get workflow runs for this PR
+ const { data: runs } = await github.rest.actions.listWorkflowRuns({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ workflow_id: 'build-ci-image.yml',
+ event: 'pull_request',
+ per_page: 10
+ });
+
+ // Find the run for this PR
+ const prRun = runs.workflow_runs.find(run =>
+ run.head_sha === context.payload.pull_request.head.sha
+ );
+
+ if (prRun) {
+ console.log(`Found workflow run: ${prRun.html_url}`);
+ console.log(`Status: ${prRun.status}, Conclusion: ${prRun.conclusion}`);
+
+ if (prRun.status === 'completed') {
+ if (prRun.conclusion === 'success') {
+ console.log('✅ Docker image build completed successfully!');
+ return;
+ } else {
+ core.setFailed(`❌ Docker image build failed with conclusion: ${prRun.conclusion}`);
+ return;
+ }
+ }
+
+ console.log(`Attempt ${attempt + 1}/${maxAttempts}: Build still running, waiting ${delaySeconds} seconds...`);
+ } else {
+ console.log(`Attempt ${attempt + 1}/${maxAttempts}: Build not started yet, waiting ${delaySeconds} seconds...`);
+ }
+
+ await new Promise(resolve => setTimeout(resolve, delaySeconds * 1000));
+ }
+
+ core.setFailed('❌ Timeout waiting for Docker image build to complete');
+
+ check-generate:
+ needs: setup
+ runs-on: ubuntu-latest
+ container:
+ image: ${{ needs.setup.outputs.image }}
+ env:
+ CARGO_HOME: /root/.cargo
+ RUSTUP_HOME: /root/.rustup
+ permissions:
+ contents: read
+ packages: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Cache cargo artifacts
+ uses: actions/cache@v4
+ with:
+ path: |
+ /root/.cargo/registry
+ /root/.cargo/git
+ /cargo-target
+ key: check-gen-cargo-${{ needs.setup.outputs.image }}-${{ hashFiles('gen/rust/Cargo.lock') }}
+ restore-keys: |
+ check-gen-cargo-${{ needs.setup.outputs.image }}-
+
+ - name: Cache uv virtualenv
+ uses: actions/cache@v4
+ with:
+ path: /uv-venv
+ key: check-gen-uv-${{ needs.setup.outputs.image }}-${{ hashFiles('gen/python/uv.lock') }}
+ restore-keys: |
+ check-gen-uv-${{ needs.setup.outputs.image }}-
+
+ - name: Run buf generation and checks
+ env:
+ SETUPTOOLS_SCM_PRETEND_VERSION: "0.0.0"
+ UV_PROJECT_ENVIRONMENT: /uv-venv
+ CARGO_TARGET_DIR: /cargo-target
+ run: |
+ git config --global --add safe.directory $GITHUB_WORKSPACE
+ cd gen/python && uv sync --all-groups --frozen && cd ../..
+ make buf
+ git diff --exit-code -- ':!gen/rust/Cargo.lock' ':!*/mocks/*' ':!*/gocachemocks/*' ':!go.mod' ':!go.sum' || \
+ (echo 'Generated files are out of date. Run `make buf` and commit changes.' && exit 1)
+ make check-crate
+
+ check-go-tidy:
+ needs: setup
+ runs-on: ubuntu-latest
+ container:
+ image: ${{ needs.setup.outputs.image }}
+ permissions:
+ contents: read
+ packages: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Run go mod tidy and check
+ run: |
+ git config --global --add safe.directory $GITHUB_WORKSPACE
+ make go-tidy
+ git diff --exit-code -- go.mod go.sum || \
+ (echo 'go.mod/go.sum are out of date. Run `go mod tidy` and commit changes.' && exit 1)
+
+ check-mocks:
+ needs: setup
+ runs-on: ubuntu-latest
+ container:
+ image: ${{ needs.setup.outputs.image }}
+ permissions:
+ contents: read
+ packages: read
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Run mockery and check
+ run: |
+ git config --global --add safe.directory $GITHUB_WORKSPACE
+ make mocks
+ git diff --exit-code -- '*/mocks/*' '*/gocachemocks/*' || \
+ (echo 'Mock files are out of date. Run `make mocks` and commit changes.' && exit 1)
diff --git a/.github/workflows/flyte-binary-v2.yml b/.github/workflows/flyte-binary-v2.yml
new file mode 100644
index 00000000000..400cfae8c71
--- /dev/null
+++ b/.github/workflows/flyte-binary-v2.yml
@@ -0,0 +1,220 @@
+name: Build & Push Flyte Single Binary Images v2
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+ cancel-in-progress: true
+
+on:
+ push:
+ branches:
+ - v2
+ pull_request:
+ branches:
+ - v2
+ workflow_dispatch:
+
+jobs:
+ test-bootstrap:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+ - name: golangci-lint
+ uses: golangci/golangci-lint-action@v3
+ with:
+ working-directory: docker/devbox-bundled/bootstrap
+ - name: Check formatting
+ working-directory: docker/devbox-bundled/bootstrap
+ run: |
+ make check-fmt
+ - name: Test
+ working-directory: docker/devbox-bundled/bootstrap
+ run: |
+ make test
+
+ build-and-push-single-binary-image:
+ runs-on: ubuntu-latest
+ needs: [test-bootstrap]
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ - name: Setup Golang caches
+ uses: actions/cache@v4
+ with:
+ path: |
+ /root/.cache/go-build
+ /root/go/pkg/mod
+ key: ${{ runner.os }}-golang-${{ hashFiles('go.sum') }}
+ restore-keys: |
+ ${{ runner.os }}-golang-
+ - name: Set versions
+ id: set_version
+ run: |
+ # TODO: The console version should be set in config and send into Dockerfile in the future
+ # echo "FLYTECONSOLE_VERSION=latest" >> $GITHUB_ENV
+ echo "FLYTE_VERSION=${{ github.sha }}" >> $GITHUB_ENV
+ - name: Prepare Image Names
+ id: image-names
+ uses: docker/metadata-action@v3
+ with:
+ images: |
+ ghcr.io/${{ github.repository_owner }}/flyte-binary-v2
+ tags: |
+ type=raw,value=nightly,enable=${{ github.event_name == 'pull_request' && github.ref == 'refs/heads/master' }}
+ type=sha,format=long
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v2
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v2
+ - name: Setup destination directories for image tarballs
+ run: |
+ mkdir -p docker/devbox-bundled/images/tar/{arm64,amd64}
+ - name: Export ARM64 Image
+ uses: docker/build-push-action@v6
+ with:
+ context: .
+ platforms: linux/arm64
+ tags: flyte-binary-v2:sandbox
+ build-args: |
+ FLYTECONSOLE_VERSION=${{ env.FLYTECONSOLE_VERSION }}
+ FLYTE_VERSION=${{ env.FLYTE_VERSION }}
+ file: Dockerfile
+ outputs: type=docker,dest=docker/devbox-bundled/images/tar/arm64/flyte-binary.tar
+ - name: Export AMD64 Image
+ uses: docker/build-push-action@v6
+ with:
+ context: .
+ platforms: linux/amd64
+ tags: flyte-binary-v2:sandbox
+ build-args: |
+ FLYTECONSOLE_VERSION=${{ env.FLYTECONSOLE_VERSION }}
+ FLYTE_VERSION=${{ env.FLYTE_VERSION }}
+ file: Dockerfile
+ outputs: type=docker,dest=docker/devbox-bundled/images/tar/amd64/flyte-binary.tar
+ - name: Upload single binary image
+ uses: actions/upload-artifact@v4
+ with:
+ name: single-binary-image
+ path: docker/devbox-bundled/images/tar
+ - name: Login to GitHub Container Registry
+ if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
+ uses: docker/login-action@v2
+ with:
+ registry: ghcr.io
+ username: "${{ secrets.FLYTE_BOT_USERNAME }}"
+ password: "${{ secrets.FLYTE_BOT_PAT }}"
+ - name: Build and push Image
+ if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
+ uses: docker/build-push-action@v6
+ with:
+ context: .
+ platforms: linux/arm64, linux/amd64
+ tags: ${{ steps.image-names.outputs.tags }}
+ build-args: |
+ FLYTECONSOLE_VERSION=${{ env.FLYTECONSOLE_VERSION }}
+ FLYTE_VERSION=${{ env.FLYTE_VERSION }}
+ file: Dockerfile
+ push: true
+
+ build-and-push-devbox-bundled-image:
+ runs-on: ubuntu-latest
+ needs: [build-and-push-single-binary-image]
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+ - uses: actions/download-artifact@v4
+ with:
+ name: single-binary-image
+ path: docker/devbox-bundled/images/tar
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v2
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v2
+ with:
+ driver-opts: image=moby/buildkit:master
+ buildkitd-flags: "--allow-insecure-entitlement security.insecure"
+ - name: Set version
+ id: set_version
+ run: |
+ echo "FLYTE_DEVBOX_VERSION=${{ github.sha }}" >> $GITHUB_ENV
+ - name: Prepare Image Names
+ id: image-names
+ uses: docker/metadata-action@v3
+ with:
+ # Push to both flyte-devbox and flyte-sandbox-v2 (legacy name)
+ # so existing users pulling the old image continue to work.
+ images: |
+ ghcr.io/${{ github.repository_owner }}/flyte-demo
+ ghcr.io/${{ github.repository_owner }}/flyte-devbox
+ ghcr.io/${{ github.repository_owner }}/flyte-sandbox-v2
+ tags: |
+ type=raw,value=nightly,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/v2' }}
+ type=raw,value=latest,enable=${{ github.event_name == 'workflow_dispatch' }}
+ type=sha,format=long,
+ - name: Login to GitHub Container Registry
+ if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
+ uses: docker/login-action@v2
+ with:
+ registry: ghcr.io
+ username: "${{ secrets.FLYTE_BOT_USERNAME }}"
+ password: "${{ secrets.FLYTE_BOT_PAT }}"
+ - name: Build CPU multi-arch image to OCI archive
+ # Produce an OCI archive locally so the GPU build below can use it as a
+ # named build context. This avoids the PR-gated push chicken-and-egg:
+ # on pull_request events we don't push to ghcr, so the GPU build can't
+ # resolve a ghcr-hosted FROM.
+ uses: docker/build-push-action@v6
+ with:
+ context: docker/devbox-bundled
+ allow: "security.insecure"
+ platforms: linux/arm64, linux/amd64
+ build-args: "FLYTE_DEVBOX_VERSION=${{ env.FLYTE_DEVBOX_VERSION }}"
+ outputs: type=oci,dest=/tmp/cpu-oci.tar
+ cache-from: type=gha,scope=demo-cpu
+ cache-to: type=gha,mode=max,scope=demo-cpu
+ - name: Extract CPU OCI layout for GPU build
+ run: |
+ mkdir -p /tmp/cpu-oci
+ tar -xf /tmp/cpu-oci.tar -C /tmp/cpu-oci
+ - name: Push CPU multi-arch image
+ if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
+ uses: docker/build-push-action@v6
+ with:
+ context: docker/devbox-bundled
+ allow: "security.insecure"
+ platforms: linux/arm64, linux/amd64
+ tags: ${{ steps.image-names.outputs.tags }}
+ build-args: "FLYTE_DEVBOX_VERSION=${{ env.FLYTE_DEVBOX_VERSION }}"
+ push: true
+ cache-from: type=gha,scope=demo-cpu
+ - name: Prepare GPU Image Names
+ id: gpu-image-names
+ uses: docker/metadata-action@v3
+ with:
+ images: |
+ ghcr.io/${{ github.repository_owner }}/flyte-devbox
+ ghcr.io/${{ github.repository_owner }}/flyte-demo
+ ghcr.io/${{ github.repository_owner }}/flyte-sandbox-v2
+ tags: |
+ type=raw,value=gpu-nightly,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/v2' }}
+ type=raw,value=gpu-latest,enable=${{ github.event_name == 'workflow_dispatch' }}
+ type=sha,format=long,prefix=gpu-
+ - name: Build and push GPU multi-arch image
+ uses: docker/build-push-action@v6
+ with:
+ context: docker/devbox-bundled
+ file: docker/devbox-bundled/Dockerfile.gpu
+ # Point Dockerfile.gpu's `FROM ${BASE_IMAGE}` at the OCI archive
+ # produced above — no registry round-trip needed.
+ build-contexts: base=oci-layout:///tmp/cpu-oci
+ allow: "security.insecure"
+ platforms: linux/arm64, linux/amd64
+ tags: ${{ steps.gpu-image-names.outputs.tags }}
+ build-args: |
+ FLYTE_DEVBOX_VERSION=${{ env.FLYTE_DEVBOX_VERSION }}
+ BASE_IMAGE=base
+ push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
+ cache-from: type=gha,scope=demo-gpu
+ cache-to: type=gha,mode=max,scope=demo-gpu
diff --git a/.github/workflows/git_town.yml b/.github/workflows/git_town.yml
new file mode 100644
index 00000000000..478ac9bfdba
--- /dev/null
+++ b/.github/workflows/git_town.yml
@@ -0,0 +1,22 @@
+name: Git Town
+
+on:
+ pull_request:
+ branches:
+ - '**'
+
+jobs:
+ git-town:
+ name: Display the branch stack
+ runs-on: ubuntu-latest
+
+ permissions:
+ contents: read
+ pull-requests: write
+
+ steps:
+ - uses: actions/checkout@v4
+ - uses: git-town/action@v1
+ with:
+ main-branch: 'main'
+ skip-single-stacks: true
diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml
new file mode 100644
index 00000000000..b86a4eb8a49
--- /dev/null
+++ b/.github/workflows/go-tests.yml
@@ -0,0 +1,74 @@
+name: Go Tests
+
+on:
+ push:
+ branches: [v2, main]
+ pull_request:
+ types: [opened, synchronize, reopened]
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ go-tests:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - component: flyteplugins
+ - component: flytestdlib
+ - component: runs
+ # Exclude runs/test/ which contains integration tests
+ test-args: $(go list ./runs/... | grep -v /test)
+ - component: executor
+ # Controller tests require envtest binaries (etcd + kube-apiserver)
+ needs-envtest: true
+ - component: flytecopilot
+ - component: dataproxy
+ - component: flyteidl2
+ - component: actions
+ - component: manager
+ name: test (${{ matrix.component }})
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Set up Go
+ uses: actions/setup-go@v5
+ with:
+ go-version: "1.24"
+ cache: true
+
+ - name: Cache embedded postgres
+ uses: actions/cache@v4
+ with:
+ path: ~/.embedded-postgres-go
+ key: embedded-postgres-v16
+
+ - name: Set up envtest
+ if: matrix.needs-envtest
+ run: |
+ ENVTEST_VERSION=$(go list -m -f "{{ .Version }}" sigs.k8s.io/controller-runtime | awk -F'[v.]' '{printf "release-%d.%d", $2, $3}')
+ go install sigs.k8s.io/controller-runtime/tools/setup-envtest@${ENVTEST_VERSION}
+ ENVTEST_K8S_VERSION=$(go list -m -f "{{ .Version }}" k8s.io/api | awk -F'[v.]' '{printf "1.%d", $3}')
+ echo "KUBEBUILDER_ASSETS=$(setup-envtest use ${ENVTEST_K8S_VERSION} --bin-dir /tmp/envtest -p path)" >> "$GITHUB_ENV"
+
+ - name: Run tests
+ run: |
+ TEST_ARGS="${{ matrix.test-args }}"
+ if [ -z "$TEST_ARGS" ]; then
+ TEST_ARGS="./${{ matrix.component }}/..."
+ fi
+ go test -race -coverprofile=coverage.out -covermode=atomic $TEST_ARGS
+
+ - name: Upload coverage
+ uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: coverage-${{ matrix.component }}
+ path: coverage.out
+ retention-days: 7
diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml
new file mode 100644
index 00000000000..fd29a9dad47
--- /dev/null
+++ b/.github/workflows/publish-npm.yml
@@ -0,0 +1,44 @@
+name: Publish TypeScript Package
+
+on:
+ push:
+ tags:
+ - 'v*'
+
+jobs:
+ publish:
+ runs-on: ubuntu-latest
+ container:
+ image: ghcr.io/flyteorg/flyte/ci:v2
+ credentials:
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+ defaults:
+ run:
+ working-directory: ./gen/ts/
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ - name: Debug Info
+ run: |
+ pwd
+ ls -la
+ - uses: actions/setup-node@v1
+ with:
+ node-version: "20.x"
+ registry-url: "https://registry.npmjs.org"
+ - name: Set version in npm package
+ run: |
+ # v1.2.3 get 1.2.3
+ VERSION=$(echo "${GITHUB_REF#refs/tags/v}")
+ VERSION=$VERSION make update_npmversion
+ shell: bash
+
+ - name: Install dependencies
+ run: npm install
+ - name: Publish
+ env:
+ NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+ run: |
+ NPM_REGISTRY_URL="https://registry.npmjs.org/"
+ npm publish --access=public --registry $NPM_REGISTRY_URL
diff --git a/.github/workflows/publish-python.yml b/.github/workflows/publish-python.yml
new file mode 100644
index 00000000000..95f92fb8416
--- /dev/null
+++ b/.github/workflows/publish-python.yml
@@ -0,0 +1,77 @@
+name: Publish Python Package
+
+on:
+ push:
+ tags:
+ - 'v*'
+ pull_request:
+ paths:
+ - '.github/workflows/publish-python.yml'
+ - 'gen/python/**'
+
+jobs:
+ publish:
+ runs-on: ubuntu-latest
+ container:
+ image: ghcr.io/flyteorg/flyte/ci:v2
+ credentials:
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+ defaults:
+ run:
+ working-directory: ./gen/python
+ env:
+ PUBLISH_TO_TESTPYPI: ${{ vars.PUBLISH_TO_TESTPYPI || 'false' }}
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: '0'
+
+ - name: Debug Info
+ run: |
+ pwd
+ ls -la
+
+ - name: Set version from tag
+ run: |
+ # Convert 1.2.3-b1 to 1.2.3b1 for Python versioning
+ VERSION="${{ github.ref_name }}"
+ VERSION_NO_V="${VERSION#v}"
+ VERSION_PY=$(echo "$VERSION_NO_V" | sed -E 's/-//')
+ # Use 0.0.0.dev0 for PR dry runs
+ if [ "${{ github.event_name }}" = "pull_request" ]; then
+ VERSION_PY="0.0.0.dev0"
+ fi
+ export SETUPTOOLS_SCM_PRETEND_VERSION="$VERSION_PY"
+ echo "SETUPTOOLS_SCM_PRETEND_VERSION=$SETUPTOOLS_SCM_PRETEND_VERSION" >> $GITHUB_ENV
+ echo "SETUPTOOLS_SCM_PRETEND_VERSION=$SETUPTOOLS_SCM_PRETEND_VERSION"
+
+ - name: Fix git safe directory
+ working-directory: .
+ run: git config --global --add safe.directory /__w/flyte/flyte
+
+ - name: Install dependencies
+ run: |
+ uv venv
+ uv pip install build twine setuptools wheel
+
+ - name: Build
+ run: |
+ uv run python -m build --wheel --installer uv
+ - name: Publish to Test
+ if: ${{ github.event_name == 'push' && env.PUBLISH_TO_TESTPYPI == 'true' }}
+ env:
+ TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+ TWINE_PASSWORD: ${{ secrets.PYPI_TEST_PASSWORD }}
+ run: |
+ REPOSITORY_URL="https://test.pypi.org/legacy/"
+ uv run python -m twine upload --repository-url "$REPOSITORY_URL" --verbose dist/*
+ - name: Publish to Pypi
+ if: ${{ github.event_name == 'push' && env.PUBLISH_TO_TESTPYPI == 'false' }}
+ env:
+ TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+ run: |
+ REPOSITORY_URL="https://upload.pypi.org/legacy/"
+ uv run python -m twine upload --repository-url "$REPOSITORY_URL" --verbose dist/*
diff --git a/.github/workflows/publish-rust.yml b/.github/workflows/publish-rust.yml
new file mode 100644
index 00000000000..834ad840e26
--- /dev/null
+++ b/.github/workflows/publish-rust.yml
@@ -0,0 +1,69 @@
+name: Publish Rust Crate
+
+on:
+ push:
+ tags:
+ - 'v*'
+
+jobs:
+ publish:
+ runs-on: ubuntu-latest
+ container:
+ image: ghcr.io/flyteorg/flyte/ci:v2
+ credentials:
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+ defaults:
+ run:
+ working-directory: ./gen/rust
+ permissions:
+ id-token: write # Required for OIDC token exchange
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Cache cargo registry
+ uses: actions/cache@v4
+ with:
+ path: ~/.cargo/registry
+ key: ${{ runner.os }}-cargo-registry-${{ hashFiles('gen/rust/Cargo.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-cargo-registry-
+
+ - name: Cache cargo git
+ uses: actions/cache@v4
+ with:
+ path: ~/.cargo/git
+ key: ${{ runner.os }}-cargo-git-${{ hashFiles('gen/rust/Cargo.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-cargo-git-
+
+ - name: Cache cargo build
+ uses: actions/cache@v4
+ with:
+ path: gen/rust/target
+ key: ${{ runner.os }}-cargo-build-${{ hashFiles('gen/rust/Cargo.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-cargo-build-
+ - name: Temporarily modify the rust toolchain version
+ run: rustup override set stable
+ - name: Output rust version for educational purposes
+ run: rustup --version
+ - name: Set Cargo.toml version from release tag
+ run: |
+ VERSION="${{ github.ref_name }}"
+ VERSION_NO_V="${VERSION#v}"
+ # Convert 1.2.3b1 to 1.2.3-b1 for semver compliance
+ VERSION_SEMVER=$(echo "$VERSION_NO_V" | sed -E 's/^([0-9]+\.[0-9]+\.[0-9]+)b([0-9]+)$/\1-b\2/')
+ sed -i.bak "s/^version = \".*\"/version = \"$VERSION_SEMVER\"/" Cargo.toml
+ rm Cargo.toml.bak
+ - name: "Package for crates.io"
+ run: cargo package --allow-dirty # publishes a package as a tarball
+ - uses: rust-lang/crates-io-auth-action@v1
+ id: auth
+ - name: Build binaries in "release" mode
+ run: cargo build -r
+ - name: "Publish to crates.io"
+ env:
+ CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
+ run: cargo publish --allow-dirty # publishes your crate as a library that can be added as a dependency
diff --git a/.github/workflows/regenerate-on-comment.yml b/.github/workflows/regenerate-on-comment.yml
new file mode 100644
index 00000000000..23cc4f51631
--- /dev/null
+++ b/.github/workflows/regenerate-on-comment.yml
@@ -0,0 +1,74 @@
+name: Regenerate and Push
+
+on:
+ issue_comment:
+ types: [created]
+
+jobs:
+ regenerate:
+ if: github.event.issue.pull_request && (contains(github.event.comment.body, '/regen') || contains(github.event.comment.body, ':repeat:'))
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ pull-requests: write
+ steps:
+ - name: Get PR details
+ id: pr
+ uses: actions/github-script@v7
+ with:
+ script: |
+ const pr = await github.rest.pulls.get({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ pull_number: context.issue.number
+ });
+ core.setOutput('number', pr.data.number);
+ core.setOutput('head_ref', pr.data.head.ref);
+ core.setOutput('head_sha', pr.data.head.sha);
+
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ ref: ${{ steps.pr.outputs.head_ref }}
+ token: ${{ secrets.GITHUB_TOKEN }}
+ fetch-depth: 0
+
+ - name: Determine Docker image
+ id: docker-image
+ run: |
+ # Check if gen.Dockerfile was modified
+ git fetch origin ${{ github.event.repository.default_branch }}
+ if git diff --name-only origin/${{ github.event.repository.default_branch }}...HEAD | grep -E '^(gen\.Dockerfile|\.github/workflows/build-ci-image\.yml)$'; then
+ echo "image=ghcr.io/flyteorg/flyte/ci:pr-${{ steps.pr.outputs.number }}" >> $GITHUB_OUTPUT
+ echo "📦 Using PR-specific image"
+ else
+ echo "image=ghcr.io/flyteorg/flyte/ci:v2" >> $GITHUB_OUTPUT
+ echo "📦 Using default v2 image"
+ fi
+
+ - name: Login to GHCR
+ run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+ - name: Pull Docker image
+ run: docker pull ${{ steps.docker-image.outputs.image }}
+
+ - name: Run generate in container
+ run: |
+ docker run --rm \
+ -v ${{ github.workspace }}:/workspace \
+ -w /workspace \
+ -e SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 \
+ -e UV_PROJECT_ENVIRONMENT=/tmp/flyte-venv \
+ ${{ steps.docker-image.outputs.image }} \
+ bash -c "git config --global --add safe.directory /workspace && cd gen/python && uv sync --all-groups --frozen && cd ../.. && make gen-local"
+
+ - name: Commit and push changes
+ run: |
+ git config --global user.name "github-actions[bot]"
+ git config --global user.email "github-actions[bot]@users.noreply.github.com"
+ git add .
+ if ! git diff --cached --quiet; then
+ git commit -m "chore: regenerate files via workflow"
+ git push origin HEAD:${{ steps.pr.outputs.head_ref }}
+ fi
+
diff --git a/.github/workflows/selfassign.yml b/.github/workflows/selfassign.yml
new file mode 100644
index 00000000000..a60bcb4c707
--- /dev/null
+++ b/.github/workflows/selfassign.yml
@@ -0,0 +1,24 @@
+# Allow users to automatically tag themselves to issues
+#
+# Usage:
+# - a github user (a member of the repo) needs to comment
+# with "#self-assign" on an issue to be assigned to them.
+#------------------------------------------------------------
+
+name: Self-assign
+on:
+ issue_comment:
+ types: created
+jobs:
+ one:
+ runs-on: ubuntu-latest
+ if: >-
+ (github.event.comment.body == '#take' ||
+ github.event.comment.body == '#self-assign')
+ steps:
+ - run: |
+ echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}"
+ curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+ -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' \
+ https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees
+ echo "Done 🔥 "
diff --git a/.github/workflows/update_site.yml b/.github/workflows/update_site.yml
new file mode 100644
index 00000000000..222c1d03c8e
--- /dev/null
+++ b/.github/workflows/update_site.yml
@@ -0,0 +1,25 @@
+name: Update version in flyte.org
+on:
+ push:
+ tags:
+ - 'v[0-9]+.[0-9]+.[0-9]+'
+
+
+jobs:
+ repository-dispatch:
+ name: Repository Dispatch
+ runs-on: ubuntu-latest
+ steps:
+ - name: Fetch version
+ id: bump
+ run: |
+ # from refs/tags/v1.2.3 get 1.2.3
+ VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
+ echo "::set-output name=version::$VERSION"
+ - name: Create an event for the release
+ uses: peter-evans/repository-dispatch@v2
+ with:
+ token: ${{ secrets.FLYTE_BOT_PAT }}
+ repository: flyteorg/flyteorg.github.io
+ event-type: release
+ client-payload: '{"tag": "${{ steps.bump.outputs.version }}"}'
diff --git a/.gitignore b/.gitignore
index 80120c50958..921e6f3b4e1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+.kube/
.idea
_build/
_bin/
@@ -27,7 +28,7 @@ dist/
dist
*.db
vendor/
-/docker/sandbox-bundled/images/tar
+/docker/devbox-bundled/images/tar
**/bin/
docs/_tags/
docs/api/flytectl
@@ -39,3 +40,11 @@ docs/_src
docs/_projects
docs/tests
empty-config.yaml
+
+**/rust/target
+**/rust/Cargo.lock
+**/flyteidl_new.egg-info
+**/flyteidl.egg-info
+**/flyteidl2.egg-info
+node_modules
+.claude/
diff --git a/.mockery.yaml b/.mockery.yaml
new file mode 100644
index 00000000000..0c9855a4939
--- /dev/null
+++ b/.mockery.yaml
@@ -0,0 +1,106 @@
+log-level: warn
+structname: '{{.InterfaceName}}'
+pkgname: mocks
+filename: "mocks.go"
+template: testify
+template-data:
+ unroll-variadic: true
+packages:
+ github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/catalog:
+ config:
+ all: true
+ dir: flyteplugins/go/tasks/pluginmachinery/catalog/mocks
+ github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/core:
+ config:
+ all: true
+ dir: flyteplugins/go/tasks/pluginmachinery/core/mocks
+ github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/io:
+ config:
+ all: true
+ dir: flyteplugins/go/tasks/pluginmachinery/io/mocks
+ github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/k8s:
+ config:
+ all: true
+ dir: flyteplugins/go/tasks/pluginmachinery/k8s/mocks
+ github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/workqueue:
+ config:
+ all: true
+ dir: flyteplugins/go/tasks/pluginmachinery/workqueue/mocks
+ github.com/flyteorg/flyte/v2/flytestdlib/cache:
+ config:
+ all: true
+ dir: flytestdlib/cache/mocks
+ github.com/flyteorg/flyte/v2/flytestdlib/storage:
+ config:
+ all: false
+ dir: flytestdlib/storage/mocks
+ interfaces:
+ ComposedProtobufStore: {}
+ Metadata: {}
+ RawStore: {}
+ ReferenceConstructor: {}
+ github.com/flyteorg/flyte/v2/gen/go/flyteidl2/connector:
+ config:
+ all: true
+ dir: gen/go/flyteidl2/connector/mocks
+ include-auto-generated: true
+ github.com/flyteorg/flyte/v2/gen/go/flyteidl2/plugins:
+ config:
+ all: true
+ dir: gen/go/flyteidl2/plugins/mocks
+ include-auto-generated: true
+ github.com/flyteorg/flyte/v2/gen/go/flyteidl2/service:
+ config:
+ all: true
+ dir: gen/go/flyteidl2/service/mocks
+ include-auto-generated: true
+ github.com/flyteorg/flyte/v2/gen/go/flyteidl2/actions/actionsconnect:
+ config:
+ all: true
+ dir: gen/go/flyteidl2/actions/actionsconnect/mocks
+ include-auto-generated: true
+ github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow/workflowconnect:
+ config:
+ all: true
+ dir: gen/go/flyteidl2/workflow/workflowconnect/mocks
+ include-auto-generated: true
+ github.com/flyteorg/flyte/v2/runs/repository/interfaces:
+ config:
+ all: true
+ dir: runs/repository/mocks
+ include-auto-generated: true
+ github.com/flyteorg/flyte/v2/flytestdlib/autorefreshcache:
+ config:
+ all: true
+ dir: flytestdlib/autorefreshcache/mocks
+ github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/webapi:
+ config:
+ all: true
+ dir: flyteplugins/go/tasks/pluginmachinery/webapi/mocks
+ github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/internal/webapi:
+ config:
+ all: true
+ dir: flyteplugins/go/tasks/pluginmachinery/internal/webapi/mocks
+ github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/secret:
+ config:
+ all: false
+ dir: flyteplugins/go/tasks/pluginmachinery/secret/mocks
+ interfaces:
+ GlobalSecretProvider:
+ SecretsInjector:
+ AWSSecretManagerClient:
+ GCPSecretManagerClient:
+ AzureKeyVaultClient:
+ MockableControllerRuntimeClient:
+ github.com/flyteorg/flyte/v2/actions/service:
+ config:
+ all: false
+ dir: actions/service/mocks
+ interfaces:
+ ActionsClientInterface:
+ github.com/eko/gocache/lib/v4/cache:
+ config:
+ all: false
+ dir: flytestdlib/cache/gocachemocks
+ interfaces:
+ CacheInterface:
diff --git a/BACKEND_README.md b/BACKEND_README.md
new file mode 100644
index 00000000000..e4c51743202
--- /dev/null
+++ b/BACKEND_README.md
@@ -0,0 +1,167 @@
+# Flyte 2 Backend
+
+This repository contains the backend infrastructure for deploying a distributed, multi-node version of Flyte 2. The backend is **Kubernetes-native** — it orchestrates workflow execution using Kubernetes primitives, scheduling tasks as pods across clusters with built-in support for multi-cluster routing, service account-based identity, and pod-level log tracking. The core architecture consists of gRPC services (QueueService, RunService, StateService) backed by PostgreSQL, using async processing and real-time streaming via PostgreSQL LISTEN/NOTIFY. See the full [Implementation Spec](https://github.com/flyteorg/flyte/blob/v2/IMPLEMENTATION_SPEC.md) for details.
+
+This repo also defines the protocol buffer schemas for Flyte's APIs and generates client libraries for Go, TypeScript, Python, and Rust. Deploy this when you need Flyte running as a scalable, distributed service across your organization.
+
+**Want to contribute?** Join us on [slack.flyte.org](https://slack.flyte.org) to get involved.
+
+## Repository Structure
+
+```
+flyte/
+├── flyteidl2/ # Protocol buffer definitions
+│ ├── common/ # Common types and utilities
+│ ├── core/ # Core Flyte types (tasks, workflows, literals)
+│ ├── imagebuilder/ # Image builder service definitions
+│ ├── logs/ # Logging types
+│ ├── secret/ # Secret management types
+│ ├── task/ # Task execution types
+│ ├── trigger/ # Trigger service definitions
+│ ├── workflow/ # Workflow types
+│ └── gen_utils/ # Language-specific generation utilities
+├── gen/ # Generated code (not checked into version control)
+│ ├── go/ # Generated Go code
+│ ├── ts/ # Generated TypeScript code
+│ ├── python/ # Generated Python code
+│ └── rust/ # Generated Rust code
+├── buf.yaml # Buf configuration
+├── buf.gen.*.yaml # Language-specific generation configs
+└── Makefile # Build automation
+```
+
+## Prerequisites
+
+- [Buf CLI](https://buf.build/docs/installation) - Protocol buffer tooling
+- Go 1.24.6 or later
+- Node.js/npm (for TypeScript generation)
+- Python 3.9+ with `uv` package manager (for Python generation)
+- Rust toolchain (for Rust generation)
+
+## Quick Start
+
+### Generate All Code
+
+To generate code for all supported languages:
+
+```bash
+make gen
+```
+
+This will:
+1. Update buf dependencies
+2. Format and lint proto files
+3. Generate code for Go, TypeScript, Python, and Rust
+4. Generate mocks for Go
+5. Run `go mod tidy`
+
+### Generate for Specific Languages Locally
+
+```bash
+make buf-go # Generate Go code only
+make buf-ts # Generate TypeScript code only
+make buf-python # Generate Python code only
+make buf-rust # Generate Rust code only
+```
+
+## Making Changes
+
+### 1. Modify Protocol Buffers
+
+Edit `.proto` files in the `flyteidl2/` directory following these guidelines:
+- Follow the existing naming conventions
+- Use proper protobuf style (snake_case for fields, PascalCase for messages)
+- Add appropriate comments and documentation
+- Ensure backward compatibility when modifying existing messages
+
+### 2. Generate Code
+
+After modifying proto files:
+
+```bash
+make docker-pull # Pull the docker image for generation
+make gen
+```
+
+### 3. Verify Your Changes
+
+Run the following to ensure everything builds correctly:
+
+```bash
+# For Go
+make go-tidy
+go build ./...
+
+# For Rust
+make build-crate
+
+# For Python
+cd gen/python && uv lock
+
+# For TypeScript
+cd gen/ts && npm install
+```
+
+### 4. Generate Mocks (Go only)
+
+If you've added or modified Go interfaces:
+
+```bash
+make gen
+```
+
+## Development Workflow
+
+1. **Format proto files**: `make buf-format`
+2. **Lint proto files**: `make buf-lint`
+3. **Generate code**: `make buf` or `make gen`
+4. **Verify builds**: Build generated code in your target language
+5. **Commit changes**: Commit both proto files and generated code
+
+## Common Tasks
+
+### Update Buf Dependencies
+
+```bash
+make gen
+```
+
+### View Available Commands
+
+```bash
+make help
+```
+
+## Versioning and Releases
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) for detailed release instructions.
+
+## Generated Code
+
+The `gen/` directory contains auto-generated code and should not be manually edited. Changes to generated code should be made by:
+1. Modifying the source `.proto` files in `flyteidl2/`
+2. Updating generation utilities in `flyteidl2/gen_utils/` if needed
+3. Running `make gen` to regenerate all code
+
+## Troubleshooting
+
+### Buf Errors
+- Ensure you have the latest version of Buf: `buf --version`
+- Update dependencies: `make buf-dep`
+- Check `buf.lock` for dependency conflicts
+
+### Go Module Issues
+- Run `make go-tidy` to clean up dependencies
+- Ensure you're using Go 1.24.6 or later
+
+### Python Generation Issues
+- Ensure `uv` is installed: `pip install uv`
+- Set the environment variable: `export SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0`
+
+### Rust Build Issues
+- Update Rust toolchain: `rustup update`
+- Navigate to `gen/rust` and run `cargo update`
+
+## Contributing
+
+We welcome contributions to Flyte 2! Please follow the guide [here](CONTRIBUTING.md).
diff --git a/CODEOWNERS b/CODEOWNERS
new file mode 100644
index 00000000000..7db33605420
--- /dev/null
+++ b/CODEOWNERS
@@ -0,0 +1,9 @@
+# This file defines code owners for the Flyte repository.
+# Each line is a file pattern followed by one or more owners.
+# Order matters - later patterns take precedence over earlier ones.
+
+# Default owners for everything in the repo
+* @flyteorg/flyte2-contributors
+
+# flyteidl2 directory requires approval from flyteidl2-contributors team
+/flyteidl2/ @flyteorg/flyteidl2-contributors
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 00000000000..e06db538c7c
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,362 @@
+# Contributing to Flyte 2
+
+Thank you for your interest in contributing to Flyte 2! This guide will help you understand the contribution workflow, testing requirements, and release process.
+
+## Table of Contents
+- [Getting Started](#getting-started)
+- [Development Workflow](#development-workflow)
+- [Making Changes](#making-changes)
+- [Testing and Verification](#testing-and-verification)
+- [Submitting Changes](#submitting-changes)
+- [Release Process](#release-process)
+- [Best Practices](#best-practices)
+
+## Getting Started
+
+### Prerequisites
+
+Before contributing, ensure you have:
+- [Buf CLI](https://buf.build/docs/installation) installed
+- Go 1.24.6 or later
+- Node.js and npm (for TypeScript)
+- Python 3.9+ with `uv` package manager
+- Rust toolchain (if working with Rust bindings)
+- Git configured with your name and email
+
+### Setting Up Your Environment
+
+1. **Fork the repository** on GitHub
+
+2. **Clone your fork**:
+ ```bash
+ git clone https://github.com/YOUR_USERNAME/flyte.git
+ cd flyte/flyte
+ ```
+
+3. **Add upstream remote**:
+ ```bash
+ git remote add upstream https://github.com/flyteorg/flyte.git
+ ```
+
+4. **Verify your setup**:
+ ```bash
+ make docker-pull # Pull the docker image for generation
+ make gen
+ ```
+
+## Development Workflow
+
+### Creating a Feature Branch
+
+Always create a new branch for your changes:
+
+```bash
+git checkout -b feature/your-feature-name
+```
+
+Use descriptive branch names:
+- `feature/add-new-task-type` - for new features
+- `fix/workflow-literal-bug` - for bug fixes
+- `docs/improve-readme` - for documentation
+- `refactor/simplify-interface` - for refactoring
+
+### Keeping Your Branch Updated
+
+Regularly sync with upstream:
+
+```bash
+git fetch upstream
+git rebase upstream/v2
+```
+
+## Making Changes
+
+### 1. Modifying Protocol Buffers
+
+When editing `.proto` files in `flyteidl2/`:
+
+**Naming Conventions:**
+- Use `snake_case` for field names: `task_id`, `execution_time`
+- Use `PascalCase` for message names: `TaskDefinition`, `WorkflowSpec`
+- Use `SCREAMING_SNAKE_CASE` for enum values: `TASK_STATE_RUNNING`
+
+**Backward Compatibility:**
+- Never change field numbers
+- Never change field types
+- Never remove required fields
+- Use `reserved` for removed fields:
+ ```protobuf
+ message Example {
+ reserved 2, 15, 9 to 11;
+ reserved "old_field_name";
+ }
+ ```
+
+**Documentation:**
+- Add clear comments for all messages, fields, and enums
+- Include examples where helpful
+- Document any constraints or validation rules
+
+**Example:**
+```protobuf
+// TaskDefinition defines the structure and configuration of a task.
+message TaskDefinition {
+ // Unique identifier for the task
+ string task_id = 1;
+
+ // Human-readable name for the task
+ string name = 2;
+
+ // Optional description explaining the task's purpose
+ string description = 3;
+}
+```
+
+### 2. Generate and Format Code
+
+After making changes:
+
+```bash
+# Format proto files
+make buf-format
+
+# Lint proto files
+make buf-lint
+
+# Generate all language bindings
+make buf
+```
+
+### 3. Update Language-Specific Utilities (if needed)
+
+If you need to customize generation for a specific language, update files in `flyteidl2/gen_utils/`:
+- `flyteidl2/gen_utils/go/` - Go-specific utilities
+- `flyteidl2/gen_utils/ts/` - TypeScript utilities
+- `flyteidl2/gen_utils/python/` - Python package configuration
+- `flyteidl2/gen_utils/rust/` - Rust crate configuration
+
+## Testing and Verification
+
+### 1. Lint and Format Checks
+
+```bash
+make buf-lint
+make buf-format
+```
+
+All proto files must pass linting without errors.
+
+### 2. Verify Generated Code Builds
+
+**Go:**
+```bash
+make buf-go
+make go-tidy
+cd gen/go
+go build ./...
+go test ./...
+```
+
+**TypeScript:**
+```bash
+make buf-ts
+cd gen/ts
+npm install
+npm run build # if there's a build script
+```
+
+**Python:**
+```bash
+make buf-python
+cd gen/python
+uv lock
+uv sync
+```
+
+**Rust:**
+```bash
+make buf-rust
+make build-crate
+```
+
+### 3. Generate Mocks (Go)
+
+If you've modified Go interfaces:
+
+```bash
+make mocks
+```
+
+### 4. Integration Testing
+
+Test your changes in a downstream project that uses flyte:
+1. Use `replace` directive in `go.mod` to point to your local changes
+2. Generate code and verify it works as expected
+3. Run integration tests in the consuming project
+
+## Submitting Changes
+
+### 1. Commit Your Changes
+
+Write clear, descriptive commit messages:
+
+```bash
+git add .
+git commit -s -m "feat: add new task execution state
+
+- Add TASK_STATE_CACHED for cached task results
+- Update state transitions to support caching
+- Add documentation for caching behavior
+
+Fixes #123"
+```
+
+**Commit message format:**
+- Use conventional commits: `feat:`, `fix:`, `docs:`, `refactor:`, `test:`
+- Include `-s` flag to sign your commits (required)
+- Reference issue numbers with `Fixes #123` or `Closes #456`
+
+### 2. Push Your Changes
+
+```bash
+git push origin feature/your-feature-name
+```
+
+### 3. Create a Pull Request
+
+1. Go to the [Flyte repository](https://github.com/flyteorg/flyte)
+2. Click "New Pull Request"
+3. Select your fork and branch
+4. Target the `v2` branch (not `main`)
+5. Fill out the PR template with:
+ - Description of changes
+ - Motivation and context
+ - Testing performed
+ - Screenshots (if UI changes)
+ - Breaking changes (if any)
+
+### 4. Code Review Process
+
+- Address reviewer feedback promptly
+- Push new commits to the same branch
+- Use `git commit --amend` for small fixes (then `git push --force`)
+- Engage in constructive discussion
+- Be patient - reviews may take time
+
+## Release Process
+
+Releases are created by maintainers following these steps:
+
+### Version Management
+
+Flyte 2 follows [Semantic Versioning 2.0.0](https://semver.org/):
+
+- **MAJOR** version (v2.0.0 → v3.0.0): Breaking changes
+- **MINOR** version (v2.1.0 → v2.2.0): New features, backward compatible
+- **PATCH** version (v2.1.1 → v2.1.2): Bug fixes, backward compatible
+
+### Creating a Release (Maintainers Only)
+
+1. **Ensure all changes are merged into `v2` branch**
+
+2. **Determine the version number** based on changes:
+ ```bash
+ # View changes since last release
+ git log v2.X.Y..HEAD --oneline
+ ```
+
+3. **Create and push a tag**:
+ ```bash
+ git checkout v2
+ git pull upstream v2
+ git tag -a v2.X.Y -m "Release v2.X.Y"
+ git push upstream v2.X.Y
+ ```
+
+4. **Create GitHub Release**:
+ - Go to [Releases](https://github.com/flyteorg/flyte/releases)
+ - Click "Draft a new release"
+ - Select the tag you just created
+ - Generate release notes
+ - Add any additional context or highlights
+ - Publish release
+
+5. **Verify Artifacts**:
+ The release triggers automated publishing to:
+ - Go modules: `github.com/flyteorg/flyte/v2`
+ - NPM: `@flyteorg/flyte`
+ - PyPI: `flyteidl2`
+ - Crates.io: `flyte`
+
+### Post-Release
+
+1. **Verify published packages**:
+ ```bash
+ # Go
+ GOPROXY=https://proxy.golang.org go list -m github.com/flyteorg/flyte/v2@v2.X.Y
+
+ # NPM
+ npm view @flyteorg/flyte@2.X.Y
+
+ # PyPI
+ pip index versions flyteidl2
+
+ # Crates.io
+ cargo search flyte
+ ```
+
+2. **Update dependent projects** with the new version
+
+3. **Announce the release** in community channels
+
+## Best Practices
+
+### Protocol Buffer Design
+
+- **Keep messages small and focused** - Single responsibility principle
+- **Use oneof for mutually exclusive fields**:
+ ```protobuf
+ message Task {
+ oneof task_type {
+ PythonTask python = 1;
+ ContainerTask container = 2;
+ }
+ }
+ ```
+- **Use optional for truly optional fields** (proto3)
+- **Use repeated for arrays/lists**
+- **Provide sensible defaults** where appropriate
+
+### Documentation
+
+- Document all public APIs
+- Include usage examples in comments
+- Keep README.md and CONTRIBUTING.md up to date
+- Add inline comments for complex logic
+
+### Git Workflow
+
+- Commit early and often
+- Keep commits atomic and focused
+- Write meaningful commit messages
+- Squash small "fix" commits before PR
+- Rebase instead of merge when updating your branch
+
+### Communication
+
+- Be respectful and professional
+- Ask questions when unclear
+- Provide context in discussions
+- Update PR descriptions as scope changes
+- Respond to reviews in a timely manner
+
+## Getting Help
+
+- **Documentation**: [Flyte Documentation](https://docs.flyte.org)
+- **Community Slack**: [Flyte Slack](https://slack.flyte.org)
+- **GitHub Issues**: [Report Issues](https://github.com/flyteorg/flyte/issues)
+- **GitHub Discussions**: [Ask Questions](https://github.com/flyteorg/flyte/discussions)
+
+## License
+
+By contributing to Flyte 2, you agree that your contributions will be licensed under the Apache License 2.0.
diff --git a/DOCKER_QUICK_START.md b/DOCKER_QUICK_START.md
new file mode 100644
index 00000000000..66ac073fbf7
--- /dev/null
+++ b/DOCKER_QUICK_START.md
@@ -0,0 +1,136 @@
+# Docker Development - Quick Start
+
+This guide gets you started with Docker-based development in under 5 minutes.
+
+## Why Docker?
+
+Using Docker ensures your local environment matches CI exactly, eliminating "works on my machine" issues.
+
+## Quick Start
+
+### 1. Pull the Image
+
+```bash
+make docker-pull
+```
+
+or
+
+```bash
+docker pull ghcr.io/flyteorg/flyte/ci:v2
+```
+
+### 2. Run Common Commands
+
+#### Generate Protocol Buffers
+```bash
+make gen
+```
+
+#### Interactive Shell
+```bash
+make docker-shell
+```
+
+### 3. Manual Docker Commands
+
+If you prefer not to use Make:
+
+```bash
+# Generate files
+docker run --rm -v $(pwd):/workspace -w /workspace \
+ ghcr.io/flyteorg/flyte/ci:v2 make gen
+
+# Interactive shell
+docker run --rm -it -v $(pwd):/workspace -w /workspace \
+ ghcr.io/flyteorg/flyte/ci:v2 bash
+```
+
+## Available Make Targets
+
+Run `make help` to see all available targets including Docker-based ones:
+
+```bash
+make help
+```
+
+Docker-specific targets:
+- `make docker-pull` - Pull the latest CI image
+- `make docker-shell` - Start interactive shell
+- `make gen` - Run code generation
+- `make build-crate` - Build Rust crate
+
+## Troubleshooting
+
+### Permission Issues
+
+If generated files have wrong ownership:
+
+```bash
+docker run --rm -v $(pwd):/workspace -w /workspace \
+ --user $(id -u):$(id -g) \
+ ghcr.io/flyteorg/flyte/ci:v2 make gen
+```
+
+### Authentication Issues
+
+Login to GitHub Container Registry:
+
+```bash
+gh auth token | docker login ghcr.io -u YOUR_USERNAME --password-stdin
+```
+
+## Updating the Docker Image
+
+### Fast Local Iteration (Recommended)
+
+If you're modifying `gen.Dockerfile`, build and test locally first:
+
+```bash
+# One command to build and generate
+make docker-dev
+
+# Or step-by-step
+make docker-build # Build image
+make docker-shell # Test interactively
+make gen # Run generation
+```
+
+This is **much faster** than waiting for PR builds!
+
+### PR Testing (After Local Testing)
+
+Once your local changes work:
+
+1. **Create a PR** with your changes
+2. **Wait for build** - A bot will comment with the PR-specific image tag
+3. **Test with PR image** to verify CI works:
+ ```bash
+ docker pull ghcr.io/flyteorg/flyte/ci:pr-123 # Use your PR number
+ docker run --rm -it -v $(pwd):/workspace -w /workspace \
+ ghcr.io/flyteorg/flyte/ci:pr-123 bash
+ ```
+4. **CI automatically uses** your new image in the PR
+
+### Workflow Comparison
+
+**Local iteration** (seconds to minutes):
+```bash
+vim gen.Dockerfile
+make docker-dev # Fast!
+# Repeat until it works
+```
+
+**PR iteration** (5-10 minutes per build):
+```bash
+git push
+# Wait for build...
+docker pull ghcr.io/flyteorg/flyte/ci:pr-123
+# Test
+```
+
+Use local iteration first, then validate with PR!
+
+## More Information
+
+See [docs/docker-image-workflow.md](docs/docker-image-workflow.md) for comprehensive documentation.
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000000..d43a445e531
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,50 @@
+# Todo(alex): We should add UI into the image when UI is done
+
+FROM --platform=${BUILDPLATFORM} golang:1.24-bookworm AS flytebuilder
+
+ARG TARGETARCH
+ENV GOARCH="${TARGETARCH}"
+ENV GOOS=linux
+ENV CGO_ENABLED=0
+
+WORKDIR /flyteorg/build
+
+COPY dataproxy dataproxy
+COPY executor executor
+COPY flytecopilot flytecopilot
+COPY flyteidl2 flyteidl2
+COPY flyteplugins flyteplugins
+COPY flytestdlib flytestdlib
+COPY gen/go gen/go
+COPY actions actions
+COPY events events
+COPY runs runs
+COPY cache_service cache_service
+COPY secret secret
+
+COPY go.mod go.sum ./
+RUN go mod download
+COPY manager manager
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/go/pkg/mod \
+ go build -v -o dist/flyte ./manager/cmd/
+
+
+FROM debian:bookworm-slim
+
+ARG FLYTE_VERSION
+ENV FLYTE_VERSION="${FLYTE_VERSION}"
+
+ENV DEBCONF_NONINTERACTIVE_SEEN=true
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install core packages
+RUN apt-get update && apt-get install --no-install-recommends --yes \
+ ca-certificates \
+ tini \
+ && rm -rf /var/lib/apt/lists/*
+
+# Copy compiled executable into image
+COPY --from=flytebuilder /flyteorg/build/dist/flyte /usr/local/bin/
+
+# Set entrypoint
+ENTRYPOINT [ "/usr/bin/tini", "-g", "--", "/usr/local/bin/flyte" ]
diff --git a/IMPLEMENTATION_SPEC.md b/IMPLEMENTATION_SPEC.md
new file mode 100644
index 00000000000..66253b59498
--- /dev/null
+++ b/IMPLEMENTATION_SPEC.md
@@ -0,0 +1,1270 @@
+# Implementation Specification: Queue, Runs, and State Services
+
+## Overview
+
+This document specifies the implementation of three gRPC services using buf connect:
+- **QueueService** - Manages execution queue for actions
+- **RunService** - Manages workflow runs and their lifecycle
+- **StateService** - Manages state persistence for actions
+
+Each service will have a simple implementation backed by PostgreSQL, using pgx for database operations, and leveraging existing flytestdlib packages for database connectivity and configuration management.
+
+## Architecture
+
+```
+flyte/
+├── queue/ # QueueService implementation
+│ ├── config/
+│ │ └── config.go # Queue-specific configuration
+│ ├── repository/
+│ │ ├── interfaces.go # Repository interfaces
+│ │ ├── postgres.go # PostgreSQL implementation
+│ │ └── models.go # Database models
+│ ├── service/
+│ │ └── queue_service.go # QueueServiceHandler implementation
+│ ├── cmd/
+│ │ └── main.go # Standalone queue service binary
+│ └── migrations/
+│ └── *.sql # Database migration files
+│
+├── runs/ # RunService implementation
+│ ├── config/
+│ │ └── config.go # Runs-specific configuration
+│ ├── repository/
+│ │ ├── interfaces.go # Repository interfaces
+│ │ ├── postgres.go # PostgreSQL implementation
+│ │ └── models.go # Database models
+│ ├── service/
+│ │ └── run_service.go # RunServiceHandler implementation
+│ ├── cmd/
+│ │ └── main.go # Standalone runs service binary
+│ └── migrations/
+│ └── *.sql # Database migration files
+│
+├── state/ # StateService implementation
+│ ├── config/
+│ │ └── config.go # State-specific configuration
+│ ├── repository/
+│ │ ├── interfaces.go # Repository interfaces
+│ │ ├── postgres.go # PostgreSQL implementation
+│ │ └── models.go # Database models
+│ ├── service/
+│ │ └── state_service.go # StateServiceHandler implementation
+│ ├── cmd/
+│ │ └── main.go # Standalone state service binary
+│ └── migrations/
+│ └── *.sql # Database migration files
+│
+└── cmd/
+ └── flyte-services/
+ └── main.go # Unified binary for all services + executor
+```
+
+## Technology Stack
+
+### Core Dependencies
+- **Protocol**: buf connect (using generated code from `gen/go/flyteidl2/workflow/workflowconnect/`)
+- **Database**: PostgreSQL
+- **Database Driver**: pgx/v5 (for raw queries)
+- **ORM**: gorm (for migrations and basic operations)
+- **Config Management**: `github.com/flyteorg/flyte/v2/flytestdlib/config`
+- **Database Utils**: `github.com/flyteorg/flyte/v2/flytestdlib/database`
+- **Logging**: `github.com/flyteorg/flyte/v2/flytestdlib/logger`
+- **CLI**: `github.com/spf13/cobra`
+
+### Service Framework
+- HTTP server using `net/http`
+- buf connect handlers mounted on HTTP server
+- Graceful shutdown support
+- Health check endpoints
+
+## Service Specifications
+
+### 1. QueueService
+
+**Location**: `queue/`
+
+**Proto Definition**: `flyteidl2/workflow/queue_service.proto`
+
+**Connect Interface**: `workflowconnect.QueueServiceHandler`
+
+#### RPCs to Implement:
+1. `EnqueueAction(EnqueueActionRequest) -> EnqueueActionResponse`
+ - Validates request
+ - Persists action to queue table
+ - Returns immediately (async processing)
+
+2. `AbortQueuedRun(AbortQueuedRunRequest) -> AbortQueuedRunResponse`
+ - Marks all actions in a run as aborted
+ - Updates abort metadata
+
+3. `AbortQueuedAction(AbortQueuedActionRequest) -> AbortQueuedActionResponse`
+ - Marks specific action as aborted
+ - Updates abort metadata
+
+#### Database Schema:
+
+```sql
+-- queue/migrations/001_create_queue_tables.sql
+
+CREATE TABLE IF NOT EXISTS queued_actions (
+ id BIGSERIAL PRIMARY KEY,
+
+ -- Action Identifier
+ org VARCHAR(255) NOT NULL,
+ project VARCHAR(255) NOT NULL,
+ domain VARCHAR(255) NOT NULL,
+ run_name VARCHAR(255) NOT NULL,
+ action_name VARCHAR(255) NOT NULL,
+
+ -- Parent reference
+ parent_action_name VARCHAR(255),
+
+ -- Action group
+ action_group VARCHAR(255),
+
+ -- Subject who created the action
+ subject VARCHAR(255),
+
+ -- Serialized action spec (protobuf or JSON)
+ action_spec JSONB NOT NULL,
+
+ -- Input/Output paths
+ input_uri TEXT NOT NULL,
+ run_output_base TEXT NOT NULL,
+
+ -- Queue metadata
+ enqueued_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ processed_at TIMESTAMP,
+
+ -- Status tracking
+ status VARCHAR(50) NOT NULL DEFAULT 'queued', -- queued, processing, completed, aborted, failed
+ abort_reason TEXT,
+ error_message TEXT,
+
+ -- Indexing
+ created_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
+
+ -- Unique constraint on action identifier
+ UNIQUE(org, project, domain, run_name, action_name)
+);
+
+CREATE INDEX idx_queued_actions_run ON queued_actions(org, project, domain, run_name);
+CREATE INDEX idx_queued_actions_status ON queued_actions(status);
+CREATE INDEX idx_queued_actions_enqueued_at ON queued_actions(enqueued_at) WHERE status = 'queued';
+CREATE INDEX idx_queued_actions_parent ON queued_actions(parent_action_name) WHERE parent_action_name IS NOT NULL;
+```
+
+#### Configuration:
+
+```go
+// queue/config/config.go
+
+type Config struct {
+ // HTTP server configuration
+ Server ServerConfig `json:"server"`
+
+ // Database configuration (reuses flytestdlib)
+ Database database.DbConfig `json:"database"`
+
+ // Queue specific settings
+ MaxQueueSize int `json:"maxQueueSize" pflag:",Maximum number of queued actions"`
+ WorkerCount int `json:"workerCount" pflag:",Number of worker goroutines for processing queue"`
+}
+
+type ServerConfig struct {
+ Port int `json:"port" pflag:",Port to bind the HTTP server"`
+ Host string `json:"host" pflag:",Host to bind the HTTP server"`
+}
+```
+
+---
+
+### 2. RunService
+
+**Location**: `runs/`
+
+**Proto Definition**: `flyteidl2/workflow/run_service.proto`
+
+**Connect Interface**: `workflowconnect.RunServiceHandler`
+
+#### RPCs to Implement:
+1. `CreateRun(CreateRunRequest) -> CreateRunResponse`
+ - Creates a new run record
+ - Initializes root action
+ - Returns run metadata
+
+2. `AbortRun(AbortRunRequest) -> AbortRunResponse`
+ - Marks run as aborted
+ - Cascades to all actions
+
+3. `GetRunDetails(GetRunDetailsRequest) -> GetRunDetailsResponse`
+ - Fetches complete run information
+ - Includes root action details
+
+4. `WatchRunDetails(WatchRunDetailsRequest) -> stream WatchRunDetailsResponse`
+ - Streams run updates
+ - Uses PostgreSQL LISTEN/NOTIFY for real-time updates
+
+5. `GetActionDetails(GetActionDetailsRequest) -> GetActionDetailsResponse`
+ - Fetches detailed action information
+ - Includes all attempts
+
+6. `WatchActionDetails(WatchActionDetailsRequest) -> stream WatchActionDetailsResponse`
+ - Streams action updates
+
+7. `GetActionData(GetActionDataRequest) -> GetActionDataResponse`
+ - Returns input/output references
+ - Does NOT load actual data (just URIs)
+
+8. `ListRuns(ListRunsRequest) -> ListRunsResponse`
+ - Paginated run listing
+ - Supports filtering by org/project/trigger
+
+9. `WatchRuns(WatchRunsRequest) -> stream WatchRunsResponse`
+ - Streams run updates matching filter
+
+10. `ListActions(ListActionsRequest) -> ListActionsResponse`
+ - Lists actions for a run
+ - Paginated
+
+11. `WatchActions(WatchActionsRequest) -> stream WatchActionsResponse`
+ - Streams action updates for a run
+ - Supports filtering
+
+12. `WatchClusterEvents(WatchClusterEventsRequest) -> stream WatchClusterEventsResponse`
+ - Streams cluster events for an action attempt
+
+13. `AbortAction(AbortActionRequest) -> AbortActionResponse`
+ - Aborts a specific action
+
+#### Database Schema:
+
+```sql
+-- runs/migrations/001_create_runs_tables.sql
+
+CREATE TYPE phase AS ENUM (
+ 'PHASE_UNSPECIFIED',
+ 'PHASE_QUEUED',
+ 'PHASE_WAITING_FOR_RESOURCES',
+ 'PHASE_INITIALIZING',
+ 'PHASE_RUNNING',
+ 'PHASE_SUCCEEDED',
+ 'PHASE_FAILED',
+ 'PHASE_ABORTED',
+ 'PHASE_TIMED_OUT'
+);
+
+CREATE TYPE action_type AS ENUM (
+ 'ACTION_TYPE_UNSPECIFIED',
+ 'ACTION_TYPE_TASK',
+ 'ACTION_TYPE_TRACE',
+ 'ACTION_TYPE_CONDITION'
+);
+
+CREATE TYPE error_kind AS ENUM (
+ 'KIND_UNSPECIFIED',
+ 'KIND_USER',
+ 'KIND_SYSTEM'
+);
+
+CREATE TABLE IF NOT EXISTS runs (
+ id BIGSERIAL PRIMARY KEY,
+
+ -- Run Identifier
+ org VARCHAR(255) NOT NULL,
+ project VARCHAR(255) NOT NULL,
+ domain VARCHAR(255) NOT NULL,
+ name VARCHAR(255) NOT NULL,
+
+ -- Root action reference
+ root_action_name VARCHAR(255) NOT NULL,
+
+ -- Trigger reference (if applicable)
+ trigger_org VARCHAR(255),
+ trigger_project VARCHAR(255),
+ trigger_domain VARCHAR(255),
+ trigger_name VARCHAR(255),
+
+ -- Run spec (serialized)
+ run_spec JSONB NOT NULL,
+
+ -- Metadata
+ created_by_principal VARCHAR(255),
+ created_by_k8s_service_account VARCHAR(255),
+
+ -- Timestamps
+ created_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
+
+ -- Unique constraint
+ UNIQUE(org, project, domain, name)
+);
+
+CREATE TABLE IF NOT EXISTS actions (
+ id BIGSERIAL PRIMARY KEY,
+
+ -- Action Identifier
+ org VARCHAR(255) NOT NULL,
+ project VARCHAR(255) NOT NULL,
+ domain VARCHAR(255) NOT NULL,
+ run_name VARCHAR(255) NOT NULL,
+ name VARCHAR(255) NOT NULL,
+
+ -- Foreign key to run
+ run_id BIGINT NOT NULL REFERENCES runs(id) ON DELETE CASCADE,
+
+ -- Parent action (for nested actions)
+ parent_action_name VARCHAR(255),
+
+ -- Action type and metadata
+ action_type action_type NOT NULL,
+ action_group VARCHAR(255),
+
+ -- Task action metadata (nullable)
+ task_org VARCHAR(255),
+ task_project VARCHAR(255),
+ task_domain VARCHAR(255),
+ task_name VARCHAR(255),
+ task_version VARCHAR(255),
+ task_type VARCHAR(255),
+ task_short_name VARCHAR(255),
+
+ -- Trace action metadata (nullable)
+ trace_name VARCHAR(255),
+
+ -- Condition action metadata (nullable)
+ condition_name VARCHAR(255),
+ condition_scope_type VARCHAR(50), -- run_id, action_id, global
+ condition_scope_value TEXT,
+
+ -- Execution metadata
+ executed_by_principal VARCHAR(255),
+ executed_by_k8s_service_account VARCHAR(255),
+
+ -- Status
+ phase phase NOT NULL DEFAULT 'PHASE_QUEUED',
+ start_time TIMESTAMP,
+ end_time TIMESTAMP,
+ attempts_count INT NOT NULL DEFAULT 0,
+
+ -- Cache status
+ cache_status VARCHAR(50),
+
+ -- Error info (if failed)
+ error_kind error_kind,
+ error_message TEXT,
+
+ -- Abort info (if aborted)
+ abort_reason TEXT,
+ aborted_by_principal VARCHAR(255),
+ aborted_by_k8s_service_account VARCHAR(255),
+
+ -- Serialized specs
+ task_spec JSONB,
+ trace_spec JSONB,
+ condition_spec JSONB,
+
+ -- Input/Output references
+ input_uri TEXT,
+ run_output_base TEXT,
+
+ -- Timestamps
+ created_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
+
+ -- Unique constraint
+ UNIQUE(org, project, domain, run_name, name)
+);
+
+CREATE TABLE IF NOT EXISTS action_attempts (
+ id BIGSERIAL PRIMARY KEY,
+
+ -- Foreign key to action
+ action_id BIGINT NOT NULL REFERENCES actions(id) ON DELETE CASCADE,
+
+ -- Attempt number (1-indexed)
+ attempt_number INT NOT NULL,
+
+ -- Phase tracking
+ phase phase NOT NULL,
+ start_time TIMESTAMP NOT NULL,
+ end_time TIMESTAMP,
+
+ -- Error info (if failed)
+ error_kind error_kind,
+ error_message TEXT,
+
+ -- Output references
+ outputs JSONB,
+
+ -- Logs
+ log_info JSONB, -- Array of TaskLog
+ logs_available BOOLEAN DEFAULT FALSE,
+
+ -- Cache status
+ cache_status VARCHAR(50),
+
+ -- Cluster assignment
+ cluster VARCHAR(255),
+
+ -- Log context (k8s pod/container info)
+ log_context JSONB,
+
+ -- Timestamps
+ created_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
+
+ -- Unique constraint
+ UNIQUE(action_id, attempt_number)
+);
+
+CREATE TABLE IF NOT EXISTS cluster_events (
+ id BIGSERIAL PRIMARY KEY,
+
+ -- Foreign key to attempt
+ attempt_id BIGINT NOT NULL REFERENCES action_attempts(id) ON DELETE CASCADE,
+
+ -- Event details
+ occurred_at TIMESTAMP NOT NULL,
+ message TEXT NOT NULL,
+
+ -- Timestamps
+ created_at TIMESTAMP NOT NULL DEFAULT NOW()
+);
+
+CREATE TABLE IF NOT EXISTS phase_transitions (
+ id BIGSERIAL PRIMARY KEY,
+
+ -- Foreign key to attempt
+ attempt_id BIGINT NOT NULL REFERENCES action_attempts(id) ON DELETE CASCADE,
+
+ -- Phase transition
+ phase phase NOT NULL,
+ start_time TIMESTAMP NOT NULL,
+ end_time TIMESTAMP,
+
+ -- Timestamps
+ created_at TIMESTAMP NOT NULL DEFAULT NOW()
+);
+
+-- Indexes for performance
+CREATE INDEX idx_runs_org_project ON runs(org, project);
+CREATE INDEX idx_runs_trigger ON runs(trigger_org, trigger_project, trigger_domain, trigger_name) WHERE trigger_name IS NOT NULL;
+CREATE INDEX idx_actions_run_id ON actions(run_id);
+CREATE INDEX idx_actions_phase ON actions(phase);
+CREATE INDEX idx_actions_parent ON actions(parent_action_name) WHERE parent_action_name IS NOT NULL;
+CREATE INDEX idx_attempts_action_id ON action_attempts(action_id);
+CREATE INDEX idx_cluster_events_attempt_id ON cluster_events(attempt_id);
+CREATE INDEX idx_phase_transitions_attempt_id ON phase_transitions(attempt_id);
+
+-- For real-time updates using LISTEN/NOTIFY
+CREATE OR REPLACE FUNCTION notify_action_change() RETURNS TRIGGER AS $$
+BEGIN
+ PERFORM pg_notify('action_updates', NEW.id::TEXT);
+ RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER action_update_trigger
+AFTER INSERT OR UPDATE ON actions
+FOR EACH ROW EXECUTE FUNCTION notify_action_change();
+
+CREATE OR REPLACE FUNCTION notify_run_change() RETURNS TRIGGER AS $$
+BEGIN
+ PERFORM pg_notify('run_updates', NEW.id::TEXT);
+ RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER run_update_trigger
+AFTER INSERT OR UPDATE ON runs
+FOR EACH ROW EXECUTE FUNCTION notify_run_change();
+```
+
+#### Configuration:
+
+```go
+// runs/config/config.go
+
+type Config struct {
+ // HTTP server configuration
+ Server ServerConfig `json:"server"`
+
+ // Database configuration
+ Database database.DbConfig `json:"database"`
+
+ // Watch/streaming settings
+ WatchBufferSize int `json:"watchBufferSize" pflag:",Buffer size for watch streams"`
+}
+
+type ServerConfig struct {
+ Port int `json:"port" pflag:",Port to bind the HTTP server"`
+ Host string `json:"host" pflag:",Host to bind the HTTP server"`
+}
+```
+
+---
+
+### 3. StateService
+
+**Location**: `state/`
+
+**Proto Definition**: `flyteidl2/workflow/state_service.proto`
+
+**Connect Interface**: `workflowconnect.StateServiceHandler`
+
+#### RPCs to Implement:
+1. `Put(stream PutRequest) -> stream PutResponse`
+ - Bidirectional streaming
+ - Persists action state (NodeStatus JSON)
+ - Returns status for each request
+
+2. `Get(stream GetRequest) -> stream GetResponse`
+ - Bidirectional streaming
+ - Retrieves action state
+
+3. `Watch(WatchRequest) -> stream WatchResponse`
+ - Server streaming
+ - Watches state changes for child actions
+ - Uses PostgreSQL LISTEN/NOTIFY
+
+#### Database Schema:
+
+```sql
+-- state/migrations/001_create_state_tables.sql
+
+CREATE TABLE IF NOT EXISTS action_states (
+ id BIGSERIAL PRIMARY KEY,
+
+ -- Action Identifier
+ org VARCHAR(255) NOT NULL,
+ project VARCHAR(255) NOT NULL,
+ domain VARCHAR(255) NOT NULL,
+ run_name VARCHAR(255) NOT NULL,
+ action_name VARCHAR(255) NOT NULL,
+
+ -- Parent reference
+ parent_action_name VARCHAR(255),
+
+ -- State data (NodeStatus as JSON)
+ state JSONB NOT NULL,
+
+ -- Phase (extracted from state for indexing/filtering)
+ phase VARCHAR(50),
+
+ -- Output URI (extracted from state)
+ output_uri TEXT,
+
+ -- Error (extracted from state)
+ error JSONB,
+
+ -- Version tracking for optimistic locking
+ version BIGINT NOT NULL DEFAULT 1,
+
+ -- Timestamps
+ created_at TIMESTAMP NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMP NOT NULL DEFAULT NOW(),
+
+ -- Unique constraint
+ UNIQUE(org, project, domain, run_name, action_name)
+);
+
+CREATE INDEX idx_action_states_parent ON action_states(parent_action_name) WHERE parent_action_name IS NOT NULL;
+CREATE INDEX idx_action_states_phase ON action_states(phase);
+
+-- For real-time updates using LISTEN/NOTIFY
+CREATE OR REPLACE FUNCTION notify_state_change() RETURNS TRIGGER AS $$
+BEGIN
+ PERFORM pg_notify('state_updates',
+ json_build_object(
+ 'action_id', NEW.id,
+ 'parent_action_name', NEW.parent_action_name
+ )::TEXT
+ );
+ RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER state_update_trigger
+AFTER INSERT OR UPDATE ON action_states
+FOR EACH ROW EXECUTE FUNCTION notify_state_change();
+```
+
+#### Configuration:
+
+```go
+// state/config/config.go
+
+type Config struct {
+ // HTTP server configuration
+ Server ServerConfig `json:"server"`
+
+ // Database configuration
+ Database database.DbConfig `json:"database"`
+
+ // State specific settings
+ MaxStateSizeBytes int `json:"maxStateSizeBytes" pflag:",Maximum size of state JSON in bytes"`
+}
+
+type ServerConfig struct {
+ Port int `json:"port" pflag:",Port to bind the HTTP server"`
+ Host string `json:"host" pflag:",Host to bind the HTTP server"`
+}
+```
+
+---
+
+## Unified Binary
+
+**Location**: `cmd/flyte-services/main.go`
+
+The unified binary provides a single entrypoint that can run:
+1. **queue** - QueueService only
+2. **runs** - RunService only
+3. **state** - StateService only
+4. **executor** - Kubernetes controller only
+5. **all** - All services together on different ports
+
+### Command Structure:
+
+```bash
+# Run queue service only
+flyte-services queue --config config.yaml
+
+# Run runs service only
+flyte-services runs --config config.yaml
+
+# Run state service only
+flyte-services state --config config.yaml
+
+# Run executor only
+flyte-services executor --config config.yaml
+
+# Run all services
+flyte-services all --config config.yaml
+```
+
+### Implementation:
+
+```go
+// cmd/flyte-services/main.go
+
+package main
+
+import (
+ "context"
+ "os"
+ "os/signal"
+ "syscall"
+
+ "github.com/spf13/cobra"
+ "github.com/flyteorg/flyte/v2/flytestdlib/config"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+
+ queuecmd "github.com/flyteorg/flyte/v2/queue/cmd"
+ runscmd "github.com/flyteorg/flyte/v2/runs/cmd"
+ statecmd "github.com/flyteorg/flyte/v2/state/cmd"
+ executorcmd "github.com/flyteorg/flyte/v2/executor/cmd"
+)
+
+var rootCmd = &cobra.Command{
+ Use: "flyte-services",
+ Short: "Flyte services unified binary",
+}
+
+func init() {
+ rootCmd.AddCommand(queuecmd.NewQueueCommand())
+ rootCmd.AddCommand(runscmd.NewRunsCommand())
+ rootCmd.AddCommand(statecmd.NewStateCommand())
+ rootCmd.AddCommand(executorcmd.NewExecutorCommand())
+ rootCmd.AddCommand(newAllCommand())
+}
+
+func main() {
+ if err := rootCmd.Execute(); err != nil {
+ os.Exit(1)
+ }
+}
+
+func newAllCommand() *cobra.Command {
+ return &cobra.Command{
+ Use: "all",
+ Short: "Run all services (queue, runs, state, executor)",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ // Setup signal handling
+ sigCh := make(chan os.Signal, 1)
+ signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+
+ // Start all services in goroutines
+ errCh := make(chan error, 4)
+
+ go func() {
+ errCh <- queuecmd.RunQueue(ctx)
+ }()
+
+ go func() {
+ errCh <- runscmd.RunRuns(ctx)
+ }()
+
+ go func() {
+ errCh <- statecmd.RunState(ctx)
+ }()
+
+ go func() {
+ errCh <- executorcmd.RunExecutor(ctx)
+ }()
+
+ // Wait for signal or error
+ select {
+ case sig := <-sigCh:
+ logger.Infof(ctx, "Received signal %v, shutting down...", sig)
+ cancel()
+ case err := <-errCh:
+ logger.Errorf(ctx, "Service error: %v", err)
+ cancel()
+ return err
+ }
+
+ return nil
+ },
+ }
+}
+```
+
+---
+
+## Database Connection Management
+
+All services use flytestdlib for database management:
+
+```go
+// Example from queue/service/queue_service.go
+
+import (
+ "github.com/flyteorg/flyte/v2/flytestdlib/database"
+ "gorm.io/gorm"
+)
+
+func initDB(ctx context.Context, cfg *database.DbConfig) (*gorm.DB, error) {
+ gormConfig := &gorm.Config{
+ // Configuration options
+ }
+
+ // Create database if it doesn't exist
+ db, err := database.CreatePostgresDbIfNotExists(ctx, gormConfig, cfg.Postgres)
+ if err != nil {
+ return nil, err
+ }
+
+ // Apply connection pool settings
+ sqlDB, err := db.DB()
+ if err != nil {
+ return nil, err
+ }
+
+ sqlDB.SetMaxIdleConns(cfg.MaxIdleConnections)
+ sqlDB.SetMaxOpenConns(cfg.MaxOpenConnections)
+ sqlDB.SetConnMaxLifetime(cfg.ConnMaxLifeTime.Duration)
+
+ return db, nil
+}
+```
+
+For pgx-specific operations (like LISTEN/NOTIFY for streaming):
+
+```go
+import (
+ "github.com/jackc/pgx/v5/pgxpool"
+)
+
+func initPgxPool(ctx context.Context, cfg *database.PostgresConfig) (*pgxpool.Pool, error) {
+ connString := fmt.Sprintf(
+ "postgres://%s:%s@%s:%d/%s?%s",
+ cfg.User,
+ resolvePassword(ctx, cfg.Password, cfg.PasswordPath),
+ cfg.Host,
+ cfg.Port,
+ cfg.DbName,
+ cfg.ExtraOptions,
+ )
+
+ return pgxpool.New(ctx, connString)
+}
+```
+
+---
+
+## Configuration Management
+
+All services use flytestdlib config:
+
+```go
+// Example from queue/cmd/main.go
+
+import (
+ "github.com/flyteorg/flyte/v2/flytestdlib/config"
+ queueconfig "github.com/flyteorg/flyte/v2/queue/config"
+)
+
+var (
+ configSection = config.MustRegisterSection("queue", &queueconfig.Config{})
+)
+
+func main() {
+ // Parse config from file and flags
+ if err := config.LoadConfig(...); err != nil {
+ panic(err)
+ }
+
+ cfg := configSection.GetConfig().(*queueconfig.Config)
+ // Use cfg...
+}
+```
+
+---
+
+## Service Implementation Pattern
+
+Each service follows this pattern:
+
+```go
+// queue/service/queue_service.go
+
+package service
+
+import (
+ "context"
+
+ "connectrpc.com/connect"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow/workflowconnect"
+ "github.com/flyteorg/flyte/v2/queue/repository"
+)
+
+type QueueService struct {
+ repo repository.Repository
+}
+
+func NewQueueService(repo repository.Repository) *QueueService {
+ return &QueueService{repo: repo}
+}
+
+// Ensure we implement the interface
+var _ workflowconnect.QueueServiceHandler = (*QueueService)(nil)
+
+func (s *QueueService) EnqueueAction(
+ ctx context.Context,
+ req *connect.Request[workflow.EnqueueActionRequest],
+) (*connect.Response[workflow.EnqueueActionResponse], error) {
+ // Validate request
+ if err := req.Msg.Validate(); err != nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, err)
+ }
+
+ // Persist to database
+ if err := s.repo.EnqueueAction(ctx, req.Msg); err != nil {
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+
+ return connect.NewResponse(&workflow.EnqueueActionResponse{}), nil
+}
+
+// ... other methods
+```
+
+---
+
+## HTTP Server Setup
+
+Each service's main.go sets up an HTTP server:
+
+```go
+// queue/cmd/main.go
+
+package cmd
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+
+ "golang.org/x/net/http2"
+ "golang.org/x/net/http2/h2c"
+
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow/workflowconnect"
+ "github.com/flyteorg/flyte/v2/queue/service"
+ "github.com/flyteorg/flyte/v2/queue/repository"
+)
+
+func RunQueue(ctx context.Context) error {
+ // Initialize database
+ db, err := initDB(ctx)
+ if err != nil {
+ return err
+ }
+
+ // Run migrations
+ if err := runMigrations(db); err != nil {
+ return err
+ }
+
+ // Create repository
+ repo := repository.NewPostgresRepository(db)
+
+ // Create service
+ svc := service.NewQueueService(repo)
+
+ // Create HTTP handler
+ mux := http.NewServeMux()
+
+ path, handler := workflowconnect.NewQueueServiceHandler(svc)
+ mux.Handle(path, handler)
+
+ // Add health check
+ mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ w.Write([]byte("OK"))
+ })
+
+ // Setup HTTP/2 support
+ addr := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port)
+ server := &http.Server{
+ Addr: addr,
+ Handler: h2c.NewHandler(mux, &http2.Server{}),
+ }
+
+ logger.Infof(ctx, "Starting Queue Service on %s", addr)
+ return server.ListenAndServe()
+}
+```
+
+---
+
+## Migration Management
+
+Each service uses golang-migrate or similar:
+
+```go
+// queue/repository/migrations.go
+
+func RunMigrations(db *gorm.DB) error {
+ return db.AutoMigrate(
+ &models.QueuedAction{},
+ )
+}
+```
+
+Or use raw SQL migrations with a migration tool.
+
+---
+
+## Repository Pattern
+
+Each service implements a repository interface:
+
+```go
+// queue/repository/interfaces.go
+
+type Repository interface {
+ EnqueueAction(ctx context.Context, req *workflow.EnqueueActionRequest) error
+ AbortQueuedRun(ctx context.Context, runID *common.RunIdentifier, reason string) error
+ AbortQueuedAction(ctx context.Context, actionID *common.ActionIdentifier, reason string) error
+ GetQueuedActions(ctx context.Context, limit int) ([]*models.QueuedAction, error)
+}
+```
+
+```go
+// queue/repository/postgres.go
+
+type PostgresRepository struct {
+ db *gorm.DB
+}
+
+func NewPostgresRepository(db *gorm.DB) Repository {
+ return &PostgresRepository{db: db}
+}
+
+func (r *PostgresRepository) EnqueueAction(ctx context.Context, req *workflow.EnqueueActionRequest) error {
+ action := &models.QueuedAction{
+ Org: req.ActionId.RunId.Org,
+ Project: req.ActionId.RunId.Project,
+ Domain: req.ActionId.RunId.Domain,
+ RunName: req.ActionId.RunId.Name,
+ ActionName: req.ActionId.Name,
+ ParentActionName: req.ParentActionName,
+ ActionGroup: req.Group,
+ Subject: req.Subject,
+ ActionSpec: req, // Will be marshaled to JSONB
+ InputUri: req.InputUri,
+ RunOutputBase: req.RunOutputBase,
+ Status: "queued",
+ }
+
+ return r.db.WithContext(ctx).Create(action).Error
+}
+```
+
+---
+
+## Streaming Implementation (Watch/Listen)
+
+For streaming RPCs, use PostgreSQL LISTEN/NOTIFY:
+
+```go
+// runs/service/run_service.go
+
+func (s *RunService) WatchRunDetails(
+ ctx context.Context,
+ req *connect.Request[workflow.WatchRunDetailsRequest],
+ stream *connect.ServerStream[workflow.WatchRunDetailsResponse],
+) error {
+ // Get initial state
+ details, err := s.repo.GetRunDetails(ctx, req.Msg.RunId)
+ if err != nil {
+ return err
+ }
+
+ if err := stream.Send(&workflow.WatchRunDetailsResponse{Details: details}); err != nil {
+ return err
+ }
+
+ // Subscribe to updates via PostgreSQL LISTEN
+ updates := make(chan *workflow.RunDetails)
+ errs := make(chan error)
+
+ go s.repo.WatchRunDetails(ctx, req.Msg.RunId, updates, errs)
+
+ for {
+ select {
+ case <-ctx.Done():
+ return nil
+ case err := <-errs:
+ return err
+ case details := <-updates:
+ if err := stream.Send(&workflow.WatchRunDetailsResponse{Details: details}); err != nil {
+ return err
+ }
+ }
+ }
+}
+```
+
+```go
+// runs/repository/postgres.go
+
+func (r *PostgresRepository) WatchRunDetails(
+ ctx context.Context,
+ runID *common.RunIdentifier,
+ updates chan<- *workflow.RunDetails,
+ errs chan<- error,
+) {
+ conn, err := r.pgxPool.Acquire(ctx)
+ if err != nil {
+ errs <- err
+ return
+ }
+ defer conn.Release()
+
+ // Listen for notifications
+ _, err = conn.Exec(ctx, "LISTEN run_updates")
+ if err != nil {
+ errs <- err
+ return
+ }
+
+ for {
+ notification, err := conn.Conn().WaitForNotification(ctx)
+ if err != nil {
+ errs <- err
+ return
+ }
+
+ // Fetch updated run details
+ details, err := r.GetRunDetails(ctx, runID)
+ if err != nil {
+ errs <- err
+ return
+ }
+
+ select {
+ case updates <- details:
+ case <-ctx.Done():
+ return
+ }
+ }
+}
+```
+
+---
+
+## Testing Strategy
+
+### Unit Tests
+- Repository layer: Mock database using testcontainers with PostgreSQL
+- Service layer: Mock repository interface
+- Use table-driven tests
+
+### Integration Tests
+- End-to-end tests with real PostgreSQL
+- Use docker-compose for local testing
+- Test streaming with multiple concurrent clients
+
+### Example:
+
+```go
+// queue/service/queue_service_test.go
+
+func TestEnqueueAction(t *testing.T) {
+ mockRepo := &mocks.Repository{}
+ svc := service.NewQueueService(mockRepo)
+
+ req := connect.NewRequest(&workflow.EnqueueActionRequest{
+ // ... populate request
+ })
+
+ mockRepo.On("EnqueueAction", mock.Anything, req.Msg).Return(nil)
+
+ resp, err := svc.EnqueueAction(context.Background(), req)
+ assert.NoError(t, err)
+ assert.NotNil(t, resp)
+}
+```
+
+---
+
+## Deployment Considerations
+
+### Configuration Files
+
+Example `config.yaml`:
+
+```yaml
+database:
+ postgres:
+ host: postgres.flyte.svc.cluster.local
+ port: 5432
+ dbname: flyte_queue
+ username: flyte
+ passwordPath: /etc/secrets/db-password
+ extraOptions: "sslmode=require"
+ maxIdleConnections: 10
+ maxOpenConnections: 100
+ connMaxLifeTime: 1h
+
+queue:
+ server:
+ host: 0.0.0.0
+ port: 8089
+ maxQueueSize: 10000
+ workerCount: 10
+
+runs:
+ server:
+ host: 0.0.0.0
+ port: 8090
+ watchBufferSize: 100
+
+state:
+ server:
+ host: 0.0.0.0
+ port: 8091
+ maxStateSizeBytes: 1048576 # 1MB
+```
+
+### Docker Compose (for local development)
+
+```yaml
+version: '3.8'
+services:
+ postgres:
+ image: postgres:15
+ environment:
+ POSTGRES_USER: flyte
+ POSTGRES_PASSWORD: flyte
+ POSTGRES_DB: flyte
+ ports:
+ - "5432:5432"
+
+ queue:
+ build: .
+ command: queue --config /etc/flyte/config.yaml
+ ports:
+ - "8089:8089"
+ depends_on:
+ - postgres
+
+ runs:
+ build: .
+ command: runs --config /etc/flyte/config.yaml
+ ports:
+ - "8090:8090"
+ depends_on:
+ - postgres
+
+ state:
+ build: .
+ command: state --config /etc/flyte/config.yaml
+ ports:
+ - "8091:8091"
+ depends_on:
+ - postgres
+```
+
+---
+
+## Implementation Phases
+
+### Phase 1: Core Infrastructure
+1. Setup project structure
+2. Implement database schemas and migrations
+3. Implement repository interfaces and PostgreSQL implementations
+4. Setup configuration management using flytestdlib
+
+### Phase 2: Service Implementation
+1. Implement QueueService
+2. Implement RunService (non-streaming RPCs first)
+3. Implement StateService (non-streaming RPCs first)
+
+### Phase 3: Streaming Support
+1. Add PostgreSQL LISTEN/NOTIFY support
+2. Implement streaming RPCs (Watch*)
+3. Test concurrent streaming clients
+
+### Phase 4: Integration
+1. Implement unified binary command structure
+2. Add health checks and metrics
+3. Integration testing
+4. Documentation
+
+### Phase 5: Production Readiness
+1. Add observability (metrics, tracing, logging)
+2. Performance testing and optimization
+3. Security audit
+4. Deployment documentation
+
+---
+
+## Open Questions
+
+1. **Migration Strategy**: Should we use golang-migrate, gorm AutoMigrate, or custom SQL scripts?
+2. **Protobuf Serialization**: Store protobuf as JSONB or use binary serialization?
+3. **Queue Processing**: Should QueueService also include worker implementation for processing queued actions?
+4. **Multi-tenancy**: How to handle org/project isolation at the database level?
+5. **Metrics**: What metrics should each service expose?
+6. **Rate Limiting**: Should services implement rate limiting per org/project?
+
+---
+
+## References
+
+- Protocol Buffers: [queue_service.proto](flyteidl2/workflow/queue_service.proto), [run_service.proto](flyteidl2/workflow/run_service.proto), [state_service.proto](flyteidl2/workflow/state_service.proto)
+- Generated Code: `gen/go/flyteidl2/workflow/workflowconnect/`
+- Database Utils: `flytestdlib/database/`
+- Config Management: `flytestdlib/config/`
+- Buf Connect: https://connectrpc.com/docs/go/getting-started
+- PostgreSQL LISTEN/NOTIFY: https://www.postgresql.org/docs/current/sql-notify.html
diff --git a/Makefile b/Makefile
new file mode 100644
index 00000000000..b983a494d86
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,308 @@
+.DEFAULT_GOAL := help
+
+CLUSTER_NAME ?= flytev2
+
+# Docker CI image configuration
+DOCKER_CI_IMAGE := ghcr.io/flyteorg/flyte/ci:v2
+
+# Environment variable flags for Docker
+DOCKER_ENV_FLAGS :=
+ifdef GITHUB_TOKEN
+ DOCKER_ENV_FLAGS += -e GITHUB_TOKEN=$(GITHUB_TOKEN)
+endif
+ifdef BUF_TOKEN
+ DOCKER_ENV_FLAGS += -e BUF_TOKEN=$(BUF_TOKEN)
+endif
+
+DOCKER_RUN := docker run --rm -v $(CURDIR):/workspace -w /workspace -e UV_PROJECT_ENVIRONMENT=/tmp/flyte-venv $(DOCKER_ENV_FLAGS) $(DOCKER_CI_IMAGE)
+
+SEPARATOR := \033[1;36m========================================\033[0m
+
+# Include common Go targets
+include go.Makefile
+
+# =============================================================================
+# Go Services Build
+# =============================================================================
+
+.PHONY: build
+build: verify ## Build all Go service binaries
+ $(MAKE) -C manager build
+ $(MAKE) -C runs build
+ $(MAKE) -C executor build
+
+# =============================================================================
+# Sandbox Commands
+# =============================================================================
+
+.PHONY: sandbox-build
+sandbox-build: ## Build and start the flyte sandbox (docker/devbox-bundled)
+ $(MAKE) -C docker/devbox-bundled build
+
+# Run in dev mode with extra arg FLYTE_DEV=True
+.PHONY: sandbox-run
+sandbox-run: ## Start the flyte sandbox without rebuilding the image
+ $(MAKE) -C docker/devbox-bundled start
+
+.PHONY: sandbox-stop
+sandbox-stop: ## Stop the flyte sandbox
+ $(MAKE) -C docker/devbox-bundled stop
+
+# =============================================================================
+# Devbox Commands
+# =============================================================================
+
+.PHONY: devbox-build
+devbox-build: ## Build and start the flyte devbox cluster (docker/devbox-bundled)
+ $(MAKE) -C docker/devbox-bundled build
+
+.PHONY: devbox-run
+devbox-run: ## Start the flyte devbox cluster without rebuilding the image
+ $(MAKE) -C docker/devbox-bundled start
+
+.PHONY: devbox-stop
+devbox-stop: ## Stop the flyte devbox cluster
+ $(MAKE) -C docker/devbox-bundled stop
+
+.PHONY: help
+help: ## Show this help message
+ @echo '🆘 Showing help message'
+ @echo 'Usage: make [target]'
+ @echo ''
+ @echo 'Available targets:'
+ @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " %-15s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+.PHONY: sep
+sep:
+ @echo "$(SEPARATOR)"
+
+# =============================================================================
+# Local Tool Commands (require buf, go, cargo, uv installed locally)
+# =============================================================================
+
+# Helper to time a step: $(call timed,step_name,command)
+define timed
+ @start=$$(date +%s); \
+ $(2); rc=$$?; \
+ elapsed=$$(( $$(date +%s) - $$start )); \
+ echo "⏱ $(1) completed in $${elapsed}s"; \
+ exit $$rc
+endef
+
+.PHONY: buf-dep
+buf-dep:
+ @echo '📦 Updating buf modules (local)'
+ $(call timed,buf-dep,buf dep update)
+ @$(MAKE) sep
+
+.PHONY: buf-format
+buf-format:
+ @echo 'Running buf format (local)'
+ $(call timed,buf-format,buf format -w)
+ @$(MAKE) sep
+
+.PHONY: buf-lint
+buf-lint:
+ @echo '🧹 Linting protocol buffer files (local)'
+ $(call timed,buf-lint,buf lint --exclude-path flytestdlib/)
+ @$(MAKE) sep
+
+.PHONY: buf-ts
+buf-ts:
+ @echo '🟦 Generating TypeScript protocol buffer files (local)'
+ $(call timed,buf-ts,buf generate --clean --template buf.gen.ts.yaml --exclude-path flytestdlib/ && \
+ cp -r flyteidl2/gen_utils/ts/* gen/ts/ && \
+ echo '📦 Installing TypeScript dependencies' && \
+ cd gen/ts && npm install --silent && \
+ echo '✅ TypeScript generation complete')
+ @$(MAKE) sep
+
+.PHONY: buf-ts-check
+buf-ts-check: buf-ts
+ @echo '🔍 Type checking generated TypeScript files'
+ $(call timed,buf-ts-check,cd gen/ts && npx tsc --noEmit || (echo '⚠️ Type checking found issues (non-fatal)' && exit 0))
+ @echo '✅ Type checking complete'
+ @$(MAKE) sep
+
+.PHONY: buf-go
+buf-go:
+ @echo '🟩 Generating Go protocol buffer files (local)'
+ $(call timed,buf-go,buf generate --clean --template buf.gen.go.yaml --exclude-path flytestdlib/)
+ @$(MAKE) sep
+
+.PHONY: buf-rust
+buf-rust:
+ @echo '🦀 Generating Rust protocol buffer files (local)'
+ $(call timed,buf-rust,buf generate --clean --template buf.gen.rust.yaml --exclude-path flytestdlib/ && \
+ cp -R flyteidl2/gen_utils/rust/* gen/rust/ && \
+ cd gen/rust && cargo update)
+ @$(MAKE) sep
+
+export SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0
+.PHONY: buf-python
+buf-python:
+ @echo '🐍 Generating Python protocol buffer files (local)'
+ $(call timed,buf-python,buf generate --clean --template buf.gen.python.yaml --exclude-path flytestdlib/ && \
+ cp flyteidl2/gen_utils/python/* gen/python/ && \
+ find gen/python -type d -exec touch {}/__init__.py \; && \
+ cd gen/python && uv lock)
+ @$(MAKE) sep
+
+.PHONY: buf
+buf: buf-dep buf-format buf-lint buf-rust buf-python buf-go buf-ts buf-ts-check
+ @echo '🛠️ Finished generating all protocol buffer files (local)'
+ @$(MAKE) sep
+
+.PHONY: go-tidy
+go-tidy:
+ @echo '🧹 Running go mod tidy (local)'
+ $(call timed,go-tidy,go mod tidy $(OUT_REDIRECT))
+ @$(MAKE) sep
+
+.PHONY: mocks
+mocks:
+ @echo "🧪 Generating go mocks (local)"
+ $(call timed,mocks,mockery $(OUT_REDIRECT))
+ @$(MAKE) sep
+
+.PHONY: gen-local
+gen-local: buf mocks go-tidy ## Generate everything using local tools (requires buf, go, cargo, uv)
+ @echo '⚡ Finished generating everything in the gen directory (local)'
+ @$(MAKE) sep
+
+.PHONY: check-crate
+check-crate: ## Verify Rust crate compiles using local cargo (faster, no artifacts)
+ @echo 'Cargo check the generated rust code (local)'
+ cd gen/rust && cargo check
+
+.PHONY: build-crate
+build-crate: ## Build Rust crate using local cargo
+ @echo 'Cargo build the generated rust code (local)'
+ cd gen/rust && cargo build
+ @$(MAKE) sep
+
+# =============================================================================
+# Package Dry-Run Commands (validate packages before publishing)
+# =============================================================================
+
+.PHONY: dry-run-npm
+dry-run-npm: ## Dry-run npm package (shows what will be published)
+ @echo '📦 NPM Package Dry Run'
+ @echo '─────────────────────────────────────────'
+ @echo '📄 Package files that will be included:'
+ @cd gen/ts && npm pack --dry-run 2>&1 | grep -v "npm notice" || true
+ @echo ''
+ @echo '📋 Package contents (from package.json "files" field):'
+ @cd gen/ts && cat package.json | grep -A 10 '"files"'
+ @echo ''
+ @echo '✅ Validation: Running npm pack to create tarball...'
+ @cd gen/ts && npm pack
+ @echo ''
+ @echo '📦 Contents of generated tarball:'
+ @cd gen/ts && tar -tzf flyteorg-flyteidl2-*.tgz | head -50
+ @echo ''
+ @echo '🧹 Cleaning up tarball...'
+ @cd gen/ts && rm -f flyteorg-flyteidl2-*.tgz
+ @echo '✅ NPM dry run complete!'
+ @$(MAKE) sep
+
+.PHONY: dry-run-python
+dry-run-python: ## Dry-run Python package (shows what will be published)
+ @echo '🐍 Python Package Dry Run'
+ @echo '─────────────────────────────────────────'
+ @echo '📦 Cleaning previous builds and venvs...'
+ @rm -rf .venv
+ @cd gen/python && rm -rf dist build *.egg-info .venv
+ @echo '📦 Building Python wheel (using Docker CI image)...'
+ @docker run --rm -v $(CURDIR):/workspace -w /workspace $(DOCKER_ENV_FLAGS) $(DOCKER_CI_IMAGE) bash -c "cd gen/python && export SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 && uv venv && uv pip install build twine setuptools wheel && uv run python -m build --wheel --installer uv"
+ @echo ''
+ @echo '✅ Running twine check for validation...'
+ @docker run --rm -v $(CURDIR):/workspace -w /workspace $(DOCKER_ENV_FLAGS) $(DOCKER_CI_IMAGE) bash -c "cd gen/python && uv pip install twine && uv run python -m twine check dist/* --strict"
+ @echo ''
+ @echo '📋 Package metadata (from pyproject.toml):'
+ @cd gen/python && grep -A 5 "^\[tool.setuptools.packages.find\]" pyproject.toml
+ @echo ''
+ @echo '📦 Contents of wheel (first 100 files):'
+ @cd gen/python && unzip -l dist/*.whl | head -100
+ @echo ''
+ @echo '📊 Wheel file size:'
+ @cd gen/python && ls -lh dist/*.whl
+ @echo ''
+ @echo '🧹 Note: build artifacts preserved for inspection in gen/python/'
+ @echo ' Run: cd gen/python && rm -rf dist/ build/ *.egg-info to clean up'
+ @echo '✅ Python dry run complete!'
+ @$(MAKE) sep
+
+.PHONY: dry-run-rust
+dry-run-rust: ## Dry-run Rust package (shows what will be published)
+ @echo '🦀 Rust Package Dry Run'
+ @echo '─────────────────────────────────────────'
+ @echo '📋 Files that will be included in crate:'
+ @cd gen/rust && cargo package --list --allow-dirty | head -100
+ @echo ''
+ @echo '📦 Creating package tarball...'
+ @cd gen/rust && cargo package --allow-dirty
+ @echo ''
+ @echo '📊 Package tarball info:'
+ @cd gen/rust && ls -lh target/package/flyteidl2-*.crate
+ @echo ''
+ @echo '📦 Contents of crate tarball (first 50 files):'
+ @cd gen/rust && tar -tzf target/package/flyteidl2-*.crate | head -50
+ @echo ''
+ @echo '✅ Validation: Running cargo build on packaged crate...'
+ @cd gen/rust && cargo build --release
+ @echo ''
+ @echo '🧹 Note: target/package/ directory preserved for inspection'
+ @echo ' Run: rm -rf gen/rust/target/package/ to clean up'
+ @echo '✅ Rust dry run complete!'
+ @$(MAKE) sep
+
+.PHONY: dry-run-all
+dry-run-all: dry-run-npm dry-run-python dry-run-rust ## Run dry-run for all packages (TypeScript, Python, Rust)
+ @echo '🎉 All package dry runs complete!'
+ @echo ''
+ @echo 'Summary:'
+ @echo ' - TypeScript: gen/ts (npm package @flyteorg/flyteidl2)'
+ @echo ' - Python: gen/python/dist/ (PyPI package flyteidl2)'
+ @echo ' - Rust: gen/rust/target/package/ (crates.io package flyteidl2)'
+ @echo ''
+ @echo 'Clean up artifacts with:'
+ @echo ' - rm -f gen/ts/*.tgz'
+ @echo ' - rm -rf gen/python/dist/'
+ @echo ' - rm -rf gen/rust/target/package/'
+ @$(MAKE) sep
+
+# =============================================================================
+# Default Commands (use Docker - no local tools required)
+# =============================================================================
+
+.PHONY: gen
+gen: ## Generate everything (uses Docker - no local tools required)
+ $(DOCKER_RUN) make gen-local
+ @echo '⚡ Finished generating everything in the gen directory (Docker)'
+ @$(MAKE) sep
+
+# Docker-based development targets
+.PHONY: docker-pull
+docker-pull: ## Pull the latest CI Docker image
+ @echo '📦 Pulling latest CI Docker image'
+ docker pull $(DOCKER_CI_IMAGE)
+ @$(MAKE) sep
+
+.PHONY: docker-build
+docker-build: ## Build Docker CI image locally (faster iteration)
+ @echo '🔨 Building Docker CI image locally (fast mode)'
+ docker build -f gen.Dockerfile -t $(DOCKER_CI_IMAGE) --cache-from $(DOCKER_CI_IMAGE) .
+ @echo '✅ Image built: $(DOCKER_CI_IMAGE)'
+ @$(MAKE) sep
+
+.PHONY: docker-shell
+docker-shell: ## Start an interactive shell in the CI Docker container
+ @echo '🐳 Starting interactive shell in CI container'
+ docker run --rm -it -v $(CURDIR):/workspace -w /workspace -e UV_PROJECT_ENVIRONMENT=/tmp/flyte-venv $(DOCKER_ENV_FLAGS) $(DOCKER_CI_IMAGE) bash
+
+# Combined workflow for fast iteration
+.PHONY: docker-dev
+docker-dev: docker-build gen ## Build local image and run generation (fast iteration)
+ @echo '✅ Local Docker image built and generation complete!'
+ @$(MAKE) sep
diff --git a/README.md b/README.md
new file mode 100644
index 00000000000..49df8dc8d69
--- /dev/null
+++ b/README.md
@@ -0,0 +1,149 @@
+# Flyte 2
+
+**Reliably orchestrate ML pipelines, models, and agents at scale — in pure Python.**
+
+[](https://pypi.org/project/flyte/)
+[](https://pypi.org/project/flyte/)
+[](LICENSE)
+[](https://flyte2intro.apps.demo.hosted.unionai.cloud/)
+[](https://www.union.ai/docs/v2/flyte/user-guide/running-locally/)
+[](https://www.union.ai/docs/v2/byoc/api-reference/flyte-sdk/)
+[](https://www.union.ai/docs/v2/byoc/api-reference/flyte-cli/)
+
+## Install
+
+```bash
+uv pip install flyte
+```
+
+For the full SDK and development tools, see the [flyte-sdk](https://github.com/flyteorg/flyte-sdk) repository.
+
+## Example
+
+```python
+import asyncio
+import flyte
+
+env = flyte.TaskEnvironment(
+ name="hello_world",
+ image=flyte.Image.from_debian_base(python_version=(3, 12)),
+)
+
+@env.task
+def calculate(x: int) -> int:
+ return x * 2 + 5
+
+@env.task
+async def main(numbers: list[int]) -> float:
+ results = await asyncio.gather(*[
+ calculate.aio(num) for num in numbers
+ ])
+ return sum(results) / len(results)
+
+if __name__ == "__main__":
+ flyte.init()
+ run = flyte.run(main, numbers=list(range(10)))
+ print(f"Result: {run.result}")
+```
+
+
+| Python | Flyte CLI |
+
+|
+
+```bash
+python hello.py
+```
+
+ |
+
+
+```bash
+flyte run hello.py main --numbers '[1,2,3]'
+```
+
+ |
+
+
+
+## Serve a Model
+
+```python
+# serving.py
+from fastapi import FastAPI
+import flyte
+from flyte.app.extras import FastAPIAppEnvironment
+
+app = FastAPI()
+env = FastAPIAppEnvironment(
+ name="my-model",
+ app=app,
+ image=flyte.Image.from_debian_base(python_version=(3, 12)).with_pip_packages(
+ "fastapi", "uvicorn"
+ ),
+)
+
+@app.get("/predict")
+async def predict(x: float) -> dict:
+ return {"result": x * 2 + 5}
+
+if __name__ == "__main__":
+ flyte.init_from_config()
+ flyte.serve(env)
+```
+
+
+| Python | Flyte CLI |
+
+|
+
+```bash
+python serving.py
+```
+
+ |
+
+
+```bash
+flyte serve serving.py env
+```
+
+ |
+
+
+
+## Local Development Experience
+
+Install the TUI for a rich local development experience:
+
+```bash
+uv pip install flyte[tui]
+```
+
+[](https://www.youtube.com/watch?v=lsfy-7DbbRM)
+
+**[Try the hosted demo in your browser](https://flyte2intro.apps.demo.hosted.unionai.cloud/)** — no installation required.
+
+## Open Source Backend
+
+The open source backend for Flyte 2 is **coming soon**. This repository will contain the Kubernetes-native backend infrastructure for deploying Flyte 2 as a distributed, multi-node service. See the [Backend README](BACKEND_README.md) for the current state of the backend, protocol buffer definitions, and contribution guide.
+
+If you need an enterprise-ready, production-grade backend for Flyte 2 today, it is available on [Union.ai](https://www.union.ai/try-flyte-2).
+
+## Learn More
+
+- **[Live Demo](https://flyte2intro.apps.demo.hosted.unionai.cloud/)** — Try Flyte 2 in your browser
+- **[Documentation](https://www.union.ai/docs/v2/flyte/user-guide/running-locally/)** — Get started running locally
+- **[SDK Reference](https://www.union.ai/docs/v2/byoc/api-reference/flyte-sdk/)** — API reference docs
+- **[CLI Reference](https://www.union.ai/docs/v2/byoc/api-reference/flyte-cli/)** — CLI docs
+- **[flyte-sdk](https://github.com/flyteorg/flyte-sdk)** — The Flyte 2 Python SDK repository
+- **[Join the Flyte 2 Production Preview](https://www.union.ai/try-flyte-2)** — Get early access
+- **[Slack](https://slack.flyte.org/)** | **[GitHub Discussions](https://github.com/flyteorg/flyte/discussions)** | **[Issues](https://github.com/flyteorg/flyte/issues)**
+
+## Contributing
+
+We welcome contributions! See the [Backend README](BACKEND_README.md) for backend development, or join us on [slack.flyte.org](https://slack.flyte.org).
+
+## License
+
+Apache 2.0 — see [LICENSE](LICENSE).
diff --git a/actions/cmd/main.go b/actions/cmd/main.go
new file mode 100644
index 00000000000..0e8da5a3002
--- /dev/null
+++ b/actions/cmd/main.go
@@ -0,0 +1,38 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/flyteorg/flyte/v2/actions"
+ actionsconfig "github.com/flyteorg/flyte/v2/actions/config"
+ "github.com/flyteorg/flyte/v2/flytestdlib/app"
+)
+
+func main() {
+ a := &app.App{
+ Name: "actions-service",
+ Short: "Actions Service for Flyte",
+ Setup: func(ctx context.Context, sc *app.SetupContext) error {
+ cfg := actionsconfig.GetConfig()
+ sc.Host = cfg.Server.Host
+ sc.Port = cfg.Server.Port
+
+ k8sClient, _, err := app.InitKubernetesClient(ctx, app.K8sConfig{
+ KubeConfig: cfg.Kubernetes.KubeConfig,
+ Namespace: cfg.Kubernetes.Namespace,
+ }, nil)
+ if err != nil {
+ return fmt.Errorf("failed to initialize Kubernetes client: %w", err)
+ }
+ sc.K8sClient = k8sClient
+ sc.Namespace = cfg.Kubernetes.Namespace
+
+ return actions.Setup(ctx, sc)
+ },
+ }
+ if err := a.Run(); err != nil {
+ os.Exit(1)
+ }
+}
diff --git a/actions/config/config.go b/actions/config/config.go
new file mode 100644
index 00000000000..46b964e0437
--- /dev/null
+++ b/actions/config/config.go
@@ -0,0 +1,68 @@
+package config
+
+import (
+ "github.com/flyteorg/flyte/v2/flytestdlib/config"
+)
+
+const configSectionKey = "actions"
+
+//go:generate pflags Config --default-var=defaultConfig
+
+var defaultConfig = &Config{
+ Server: ServerConfig{
+ Port: 8091,
+ Host: "0.0.0.0",
+ },
+ Kubernetes: KubernetesConfig{
+ Namespace: "flyte",
+ },
+ WatchBufferSize: 100,
+ WatchWorkers: 10,
+ RunServiceURL: "http://localhost:8090",
+ // 8M slots × 8 bytes/pointer = 64 MB; can track ~8M unique actions.
+ RecordFilterSize: 1 << 23,
+}
+
+var configSection = config.MustRegisterSection(configSectionKey, defaultConfig)
+
+// Config holds the configuration for the Actions service
+type Config struct {
+ // HTTP server configuration
+ Server ServerConfig `json:"server"`
+
+ // Kubernetes configuration
+ Kubernetes KubernetesConfig `json:"kubernetes"`
+
+ // WatchBufferSize is the buffer size for each worker's event channel.
+ WatchBufferSize int `json:"watchBufferSize" pflag:",Buffer size for watch channels"`
+
+ // WatchWorkers is the number of parallel event-processing goroutines.
+ // Events for the same TaskAction are always routed to the same worker to preserve ordering.
+ WatchWorkers int `json:"watchWorkers" pflag:",Number of parallel worker goroutines for processing watch events"`
+
+ // RunServiceURL is the base URL for the internal run service.
+ RunServiceURL string `json:"runServiceUrl" pflag:",Base URL of the internal run service"`
+
+ // RecordFilterSize is the size of the bloom filter used to deduplicate RecordAction calls.
+ RecordFilterSize int `json:"recordFilterSize" pflag:",Size of the oppo bloom filter for deduplicating RecordAction calls"`
+}
+
+// ServerConfig holds HTTP server configuration
+type ServerConfig struct {
+ Port int `json:"port" pflag:",Port to bind the HTTP server"`
+ Host string `json:"host" pflag:",Host to bind the HTTP server"`
+}
+
+// KubernetesConfig holds Kubernetes client configuration
+type KubernetesConfig struct {
+ // Namespace where TaskAction CRs are located
+ Namespace string `json:"namespace" pflag:",Kubernetes namespace for TaskAction CRs"`
+
+ // KubeConfig path (optional - if empty, uses in-cluster config)
+ KubeConfig string `json:"kubeconfig" pflag:",Path to kubeconfig file (optional)"`
+}
+
+// GetConfig returns the parsed actions configuration
+func GetConfig() *Config {
+ return configSection.GetConfig().(*Config)
+}
diff --git a/actions/k8s/client.go b/actions/k8s/client.go
new file mode 100644
index 00000000000..9db00c5ef7e
--- /dev/null
+++ b/actions/k8s/client.go
@@ -0,0 +1,919 @@
+package k8s
+
+import (
+ "context"
+ "fmt"
+ "hash/fnv"
+ "strings"
+ "sync"
+
+ "connectrpc.com/connect"
+ "google.golang.org/protobuf/proto"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/watch"
+ "k8s.io/client-go/kubernetes/scheme"
+ toolscache "k8s.io/client-go/tools/cache"
+ ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+
+ executorv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+ "github.com/flyteorg/flyte/v2/executor/pkg/plugin"
+ "github.com/flyteorg/flyte/v2/flytestdlib/fastcheck"
+ k8sutil "github.com/flyteorg/flyte/v2/flytestdlib/k8s"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/actions"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/task"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow/workflowconnect"
+)
+
+// ActionUpdate represents an update to a TaskAction
+type ActionUpdate struct {
+ ActionID *common.ActionIdentifier
+ ParentActionName string
+ StateJSON string
+ Phase common.ActionPhase
+ OutputUri string
+ IsDeleted bool
+ TaskType string
+ ShortName string
+}
+
+const labelTerminalStatusRecorded = "flyte.org/terminal-status-recorded"
+
+// ActionsClient handles all etcd/K8s TaskAction CR operations for the Actions service.
+type ActionsClient struct {
+ k8sClient client.WithWatch
+ sharedCache ctrlcache.Cache
+ namespace string
+ bufferSize int
+ runClient workflowconnect.InternalRunServiceClient
+ // recordedFilter deduplicates RecordAction calls across watch reconnects.
+ recordedFilter fastcheck.Filter
+
+ // Watch management
+ mu sync.RWMutex
+ // Map parent action name to subscriber channels.
+ // Multiple callers may watch the same parent action concurrently.
+ // TODO: add a prometheus counter for dropped updates when metrics are wired up
+ subscribers map[string]map[chan *ActionUpdate]struct{}
+ stopCh chan struct{}
+ watching bool
+
+ // Worker pool: numWorkers goroutines each own one channel.
+ // Events are sharded by TaskAction name so per-resource ordering is preserved.
+ numWorkers int
+ workerChs []chan watch.Event
+}
+
+// NewActionsClient creates a new Kubernetes-based actions client.
+func NewActionsClient(k8sClient client.WithWatch, sharedCache ctrlcache.Cache, namespace string, bufferSize int, numWorkers int, runClient workflowconnect.InternalRunServiceClient, recordFilterSize int, scope promutils.Scope) *ActionsClient {
+ if numWorkers <= 0 {
+ numWorkers = 1
+ }
+ c := &ActionsClient{
+ k8sClient: k8sClient,
+ sharedCache: sharedCache,
+ namespace: namespace,
+ bufferSize: bufferSize,
+ numWorkers: numWorkers,
+ runClient: runClient,
+ subscribers: make(map[string]map[chan *ActionUpdate]struct{}),
+ }
+
+ if recordFilterSize > 0 {
+ filter, err := fastcheck.NewOppoBloomFilter(recordFilterSize, scope.NewSubScope("actions_filter"))
+ if err != nil {
+ logger.Warnf(context.Background(), "Failed to create record filter (size=%d): %v; proceeding without dedup", recordFilterSize, err)
+ } else {
+ c.recordedFilter = filter
+ }
+ }
+
+ return c
+}
+
+// Enqueue creates a TaskAction CR in etcd (via the K8s API).
+func (c *ActionsClient) Enqueue(ctx context.Context, action *actions.Action, runSpec *task.RunSpec) error {
+ actionID := action.ActionId
+ logger.Infof(ctx, "Enqueuing action: %s/%s/%s/%s",
+ actionID.Run.Project, actionID.Run.Domain,
+ actionID.Run.Name, actionID.Name)
+
+ isRoot := action.ParentActionName == nil || *action.ParentActionName == ""
+
+ switch action.GetSpec().(type) {
+ case *actions.Action_Task:
+ taskActionName := buildTaskActionName(actionID)
+ namespace := buildNamespace(actionID.Run)
+ if err := k8sutil.EnsureNamespaceExists(ctx, c.k8sClient, namespace); err != nil {
+ return fmt.Errorf("failed to ensure namespace %s: %w", namespace, err)
+ }
+ taskAction := &executorv1.TaskAction{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: taskActionName,
+ Namespace: namespace,
+ Labels: map[string]string{
+ "flyte.org/project": actionID.Run.Project,
+ "flyte.org/domain": actionID.Run.Domain,
+ "flyte.org/run": actionID.Run.Name,
+ "flyte.org/action": actionID.Name,
+ "flyte.org/action-type": "task", // TODO: derive from action.Spec
+ "flyte.org/is-root": fmt.Sprintf("%t", isRoot),
+ },
+ },
+ Spec: executorv1.TaskActionSpec{},
+ }
+ var parentTaskAction *executorv1.TaskAction
+ // Set OwnerReference to parent so K8s cascades deletion to children.
+ if !isRoot {
+ parentID := &common.ActionIdentifier{
+ Run: actionID.Run,
+ Name: *action.ParentActionName,
+ }
+ parentName := buildTaskActionName(parentID)
+
+ parent := &executorv1.TaskAction{}
+ if err := c.k8sClient.Get(ctx, client.ObjectKey{Name: parentName, Namespace: namespace}, parent); err != nil {
+ return fmt.Errorf("failed to get parent TaskAction %s: %w", parentName, err)
+ }
+ parentTaskAction = parent
+
+ blockOwnerDeletion := true
+ taskAction.OwnerReferences = []metav1.OwnerReference{
+ {
+ APIVersion: "flyte.org/v1",
+ Kind: "TaskAction",
+ Name: parent.Name,
+ UID: parent.UID,
+ BlockOwnerDeletion: &blockOwnerDeletion,
+ },
+ }
+ // For child actions, inherit parent's run context
+ inheritRunContextFromParentTaskAction(taskAction, parentTaskAction)
+ } else {
+ // For root action, apply the RunSpec to TaskAction
+ applyRunSpecToTaskAction(taskAction, runSpec)
+ }
+
+ // Build and set the ActionSpec for the executor.
+ actionSpec := buildActionSpec(action, runSpec)
+ if err := taskAction.Spec.SetActionSpec(actionSpec); err != nil {
+ return fmt.Errorf("failed to set action spec: %w", err)
+ }
+ taskAction.Spec.CacheKey = extractTaskCacheKey(action)
+
+ // Embed the inline TaskTemplate if present.
+ if err := embedTaskTemplate(action, taskAction); err != nil {
+ return fmt.Errorf("failed to embed task template: %w", err)
+ }
+
+ if err := c.k8sClient.Create(ctx, taskAction); err != nil {
+ return fmt.Errorf("failed to create TaskAction CR %s: %w", taskActionName, err)
+ }
+
+ logger.Infof(ctx, "Created TaskAction CR: %s", taskActionName)
+ case *actions.Action_Trace:
+ // For trace action, we only need to record result in RunService as it is already computed in SDK runtime
+ recordReq := &workflow.RecordActionRequest{
+ ActionId: actionID,
+ Parent: action.GetParentActionName(),
+ InputUri: action.GetInputUri(),
+ Group: action.GetGroup(),
+ Subject: action.GetSubject(),
+ Spec: &workflow.RecordActionRequest_Trace{
+ Trace: action.GetTrace(),
+ },
+ }
+ _, err := c.runClient.RecordAction(ctx, connect.NewRequest(recordReq))
+ if err != nil {
+ return err
+ }
+
+ // Notify SDK runtime informer
+ c.notifySubscribers(ctx, &ActionUpdate{
+ ActionID: actionID,
+ ParentActionName: action.GetParentActionName(),
+ Phase: common.ActionPhase_ACTION_PHASE_SUCCEEDED,
+ OutputUri: action.GetTrace().GetOutputs().GetOutputUri(),
+ TaskType: "trace",
+ })
+ }
+
+ return nil
+}
+
+// AbortAction deletes a TaskAction CR from etcd.
+// K8s cascades the deletion to all descendants via OwnerReferences.
+func (c *ActionsClient) AbortAction(ctx context.Context, actionID *common.ActionIdentifier, reason *string) error {
+ taskActionName := buildTaskActionName(actionID)
+ logger.Infof(ctx, "Aborting action %s (reason: %v)", taskActionName, reason)
+
+ taskAction := &executorv1.TaskAction{}
+ if err := c.k8sClient.Get(ctx, client.ObjectKey{Name: taskActionName, Namespace: buildNamespace(actionID.Run)}, taskAction); err != nil {
+ return fmt.Errorf("failed to get TaskAction %s: %w", taskActionName, err)
+ }
+
+ if err := c.k8sClient.Delete(ctx, taskAction); err != nil {
+ return fmt.Errorf("failed to delete TaskAction %s: %w", taskActionName, err)
+ }
+
+ logger.Infof(ctx, "Deleted TaskAction %s (descendants will be cascade deleted by K8s)", taskActionName)
+ return nil
+}
+
+// GetState retrieves the state JSON for a TaskAction
+func (c *ActionsClient) GetState(ctx context.Context, actionID *common.ActionIdentifier) (string, error) {
+ taskActionName := buildTaskActionName(actionID)
+
+ taskAction := &executorv1.TaskAction{}
+ if err := c.k8sClient.Get(ctx, client.ObjectKey{
+ Name: taskActionName,
+ Namespace: buildNamespace(actionID.Run),
+ }, taskAction); err != nil {
+ return "", fmt.Errorf("failed to get TaskAction %s: %w", taskActionName, err)
+ }
+
+ return taskAction.Status.StateJSON, nil
+}
+
+// PutState updates the state JSON and latest attempt metadata for a TaskAction.
+func (c *ActionsClient) PutState(ctx context.Context, actionID *common.ActionIdentifier, attempt uint32, status *workflow.ActionStatus, stateJSON string) error {
+ taskActionName := buildTaskActionName(actionID)
+
+ // Get current TaskAction
+ taskAction := &executorv1.TaskAction{}
+ if err := c.k8sClient.Get(ctx, client.ObjectKey{
+ Name: taskActionName,
+ Namespace: buildNamespace(actionID.Run),
+ }, taskAction); err != nil {
+ return fmt.Errorf("failed to get TaskAction %s: %w", taskActionName, err)
+ }
+
+ // Skip update if the stateJSON does not change
+ if taskAction.Status.StateJSON == stateJSON {
+ return nil
+ }
+
+ // Update state JSON
+ taskAction.Status.StateJSON = stateJSON
+ if status != nil {
+ taskAction.Status.Attempts = status.GetAttempts()
+ taskAction.Status.CacheStatus = status.GetCacheStatus()
+ }
+
+ // Update status subresource
+ if err := c.k8sClient.Status().Update(ctx, taskAction); err != nil {
+ return fmt.Errorf("failed to update TaskAction status %s: %w", taskActionName, err)
+ }
+
+ logger.Infof(ctx, "Updated state for TaskAction: %s", taskActionName)
+ return nil
+}
+
+// ListRunActions lists all TaskActions belonging to a run.
+func (c *ActionsClient) ListRunActions(ctx context.Context, runID *common.RunIdentifier) ([]*executorv1.TaskAction, error) {
+ taskActionList := &executorv1.TaskActionList{}
+ listOpts := []client.ListOption{
+ client.InNamespace(buildNamespace(runID)),
+ client.MatchingLabels{
+ "flyte.org/project": runID.Project,
+ "flyte.org/domain": runID.Domain,
+ "flyte.org/run": runID.Name,
+ },
+ }
+
+ if err := c.k8sClient.List(ctx, taskActionList, listOpts...); err != nil {
+ return nil, fmt.Errorf("failed to list TaskActions for run: %w", err)
+ }
+
+ result := make([]*executorv1.TaskAction, len(taskActionList.Items))
+ for i := range taskActionList.Items {
+ result[i] = &taskActionList.Items[i]
+ }
+ return result, nil
+}
+
+// ListChildActions lists all TaskActions that are children of the given parent action
+func (c *ActionsClient) ListChildActions(ctx context.Context, parentActionID *common.ActionIdentifier) ([]*executorv1.TaskAction, error) {
+ // List all TaskActions in the same run
+ taskActionList := &executorv1.TaskActionList{}
+ listOpts := []client.ListOption{
+ client.InNamespace(buildNamespace(parentActionID.Run)),
+ client.MatchingLabels{
+ "flyte.org/project": parentActionID.Run.Project,
+ "flyte.org/domain": parentActionID.Run.Domain,
+ "flyte.org/run": parentActionID.Run.Name,
+ },
+ }
+
+ if err := c.k8sClient.List(ctx, taskActionList, listOpts...); err != nil {
+ return nil, fmt.Errorf("failed to list TaskActions: %w", err)
+ }
+
+ // Filter for the parent and its children
+ var result []*executorv1.TaskAction
+ for i := range taskActionList.Items {
+ action := &taskActionList.Items[i]
+ // Include the parent action itself
+ if action.Spec.ActionName == parentActionID.Name {
+ result = append(result, action)
+ continue
+ }
+ // Include direct children
+ if action.Spec.ParentActionName != nil && *action.Spec.ParentActionName == parentActionID.Name {
+ result = append(result, action)
+ }
+ }
+
+ return result, nil
+}
+
+// GetTaskAction retrieves a specific TaskAction
+func (c *ActionsClient) GetTaskAction(ctx context.Context, actionID *common.ActionIdentifier) (*executorv1.TaskAction, error) {
+ taskActionName := buildTaskActionName(actionID)
+
+ taskAction := &executorv1.TaskAction{}
+ if err := c.k8sClient.Get(ctx, client.ObjectKey{
+ Name: taskActionName,
+ Namespace: buildNamespace(actionID.Run),
+ }, taskAction); err != nil {
+ return nil, fmt.Errorf("failed to get TaskAction %s: %w", taskActionName, err)
+ }
+
+ return taskAction, nil
+}
+
+// Subscribe creates a new subscription channel for action updates for specified parent action name
+func (c *ActionsClient) Subscribe(parentActionName string) chan *ActionUpdate {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ ch := make(chan *ActionUpdate, c.bufferSize)
+ if c.subscribers[parentActionName] == nil {
+ c.subscribers[parentActionName] = make(map[chan *ActionUpdate]struct{})
+ }
+ c.subscribers[parentActionName][ch] = struct{}{}
+ return ch
+}
+
+// Unsubscribe removes the given channel from the subscription list for the parent action name
+func (c *ActionsClient) Unsubscribe(parentActionName string, ch chan *ActionUpdate) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ if channels, ok := c.subscribers[parentActionName]; ok {
+ delete(channels, ch)
+ close(ch)
+ if len(channels) == 0 {
+ delete(c.subscribers, parentActionName)
+ }
+ }
+}
+
+// StartWatching starts watching TaskAction resources and notifies all subscribers.
+// It requires a shared controller-runtime cache.
+func (c *ActionsClient) StartWatching(ctx context.Context) error {
+ c.mu.Lock()
+ if c.watching {
+ c.mu.Unlock()
+ return nil
+ }
+ c.watching = true
+ c.stopCh = make(chan struct{})
+ stopCh := c.stopCh // capture before releasing so workers reference the right channel
+ c.workerChs = make([]chan watch.Event, c.numWorkers)
+ for i := range c.workerChs {
+ c.workerChs[i] = make(chan watch.Event, c.bufferSize)
+ go c.worker(ctx, c.workerChs[i], stopCh)
+ }
+ c.mu.Unlock()
+
+ logger.Infof(ctx, "Starting TaskAction watcher for namespace: %s (workers: %d)", c.namespace, c.numWorkers)
+
+ if c.sharedCache == nil {
+ return fmt.Errorf("shared cache is required for TaskAction informer")
+ }
+
+ return c.setupInformer(ctx)
+}
+
+func (c *ActionsClient) setupInformer(ctx context.Context) error {
+ informer, err := c.sharedCache.GetInformer(ctx, &executorv1.TaskAction{})
+ if err != nil {
+ return fmt.Errorf("failed to get TaskAction informer: %w", err)
+ }
+
+ _, err = informer.AddEventHandler(toolscache.ResourceEventHandlerFuncs{
+ AddFunc: func(obj interface{}) {
+ taskAction, ok := obj.(*executorv1.TaskAction)
+ if !ok || c.shouldSkipTaskAction(taskAction) {
+ return
+ }
+ c.dispatchEvent(taskAction, watch.Added)
+ },
+ UpdateFunc: func(_, newObj interface{}) {
+ taskAction, ok := newObj.(*executorv1.TaskAction)
+ if !ok || c.shouldSkipTaskAction(taskAction) {
+ return
+ }
+ c.dispatchEvent(taskAction, watch.Modified)
+ },
+ DeleteFunc: func(obj interface{}) {
+ // The informer may deliver a DeletedFinalStateUnknown tombstone
+ // when a delete event was missed; unwrap it first.
+ if tombstone, ok := obj.(toolscache.DeletedFinalStateUnknown); ok {
+ obj = tombstone.Obj
+ }
+ taskAction, ok := obj.(*executorv1.TaskAction)
+ if !ok {
+ return
+ }
+ c.dispatchEvent(taskAction, watch.Deleted)
+ },
+ })
+ if err != nil {
+ return fmt.Errorf("failed to add TaskAction informer handler: %w", err)
+ }
+
+ return nil
+}
+
+// worker drains its event channel and handles each event inline.
+// Each worker owns a disjoint shard of TaskAction names, so per-resource
+// event ordering is preserved across the pool.
+func (c *ActionsClient) worker(ctx context.Context, ch <-chan watch.Event, stopCh <-chan struct{}) {
+ for {
+ select {
+ case <-stopCh:
+ return
+ case <-ctx.Done():
+ return
+ case event, ok := <-ch:
+ if !ok {
+ return
+ }
+ c.handleWatchEvent(ctx, event)
+ }
+ }
+}
+
+// dispatchEvent routes an informer event to the worker responsible for the
+// TaskAction, using FNV-32a hashing for consistent sharding.
+func (c *ActionsClient) dispatchEvent(taskAction *executorv1.TaskAction, eventType watch.EventType) {
+ if taskAction == nil {
+ return
+ }
+
+ h := fnv.New32a()
+ _, _ = h.Write([]byte(taskAction.Name)) // FNV Write never returns an error
+ shard := h.Sum32() % uint32(c.numWorkers)
+ c.workerChs[shard] <- watch.Event{Type: eventType, Object: taskAction.DeepCopy()}
+}
+
+func (c *ActionsClient) handleWatchEvent(ctx context.Context, event watch.Event) {
+ taskAction, ok := event.Object.(*executorv1.TaskAction)
+ if !ok {
+ logger.Warnf(ctx, "received non-TaskAction object in watch event: %T", event.Object)
+ return
+ }
+
+ c.handleTaskActionEvent(ctx, taskAction, event.Type)
+}
+
+func (c *ActionsClient) handleTaskActionEvent(ctx context.Context, taskAction *executorv1.TaskAction, eventType watch.EventType) {
+ update := buildActionUpdate(ctx, taskAction, eventType)
+ if update == nil {
+ return
+ }
+
+ c.notifySubscribers(ctx, update)
+ c.notifyRunService(ctx, taskAction, update, eventType)
+}
+
+func buildActionUpdate(ctx context.Context, taskAction *executorv1.TaskAction, eventType watch.EventType) *ActionUpdate {
+ var parentName string
+ if taskAction.Spec.ParentActionName != nil {
+ parentName = *taskAction.Spec.ParentActionName
+ }
+
+ // Determine short name: use spec.ShortName if set, otherwise extract from template ID
+ shortName := taskAction.Spec.ShortName
+ if shortName == "" && len(taskAction.Spec.TaskTemplate) > 0 {
+ shortName = extractShortNameFromTemplate(taskAction.Spec.TaskTemplate)
+ }
+
+ phase := GetPhaseFromConditions(taskAction)
+ if eventType == watch.Deleted {
+ phase = common.ActionPhase_ACTION_PHASE_ABORTED
+ }
+
+ return &ActionUpdate{
+ ActionID: &common.ActionIdentifier{
+ Run: &common.RunIdentifier{
+ Project: taskAction.Spec.Project,
+ Domain: taskAction.Spec.Domain,
+ Name: taskAction.Spec.RunName,
+ },
+ Name: taskAction.Spec.ActionName,
+ },
+ ParentActionName: parentName,
+ StateJSON: taskAction.Status.StateJSON,
+ Phase: phase,
+ OutputUri: buildOutputUri(ctx, taskAction),
+ IsDeleted: eventType == watch.Deleted,
+ TaskType: taskAction.Spec.TaskType,
+ ShortName: shortName,
+ }
+}
+
+func (c *ActionsClient) shouldSkipTaskAction(taskAction *executorv1.TaskAction) bool {
+ if taskAction == nil {
+ return true
+ }
+ return taskAction.GetLabels()[labelTerminalStatusRecorded] == "true"
+}
+
+// notifySubscribers sends an update to all subscribers
+func (c *ActionsClient) notifySubscribers(ctx context.Context, update *ActionUpdate) {
+ c.mu.RLock()
+ defer c.mu.RUnlock()
+
+ for ch := range c.subscribers[update.ParentActionName] {
+ select {
+ case ch <- update:
+ default:
+ logger.Warnf(ctx, "subscriber channel full, dropping update for parent action: %s", update.ParentActionName)
+ }
+ }
+}
+
+// notifyRunService forwards a watch event to the internal run service.
+// On ADDED events it calls RecordAction to create the DB record.
+// On all events it calls UpdateActionStatus (when phase is meaningful) to update the actions table.
+func (c *ActionsClient) notifyRunService(ctx context.Context, taskAction *executorv1.TaskAction, update *ActionUpdate, eventType watch.EventType) {
+ if c.runClient == nil {
+ return
+ }
+
+ // On ADDED: create the action record in the DB (deduplicated via bloom filter).
+ if eventType == watch.Added {
+ actionKey := []byte(buildTaskActionName(update.ActionID))
+ isDuplicate := c.recordedFilter != nil && c.recordedFilter.Contains(ctx, actionKey)
+ if isDuplicate {
+ logger.Debugf(ctx, "Skipping duplicate RecordAction for %s", update.ActionID.Name)
+ } else {
+ recordReq := &workflow.RecordActionRequest{
+ ActionId: update.ActionID,
+ Parent: update.ParentActionName,
+ InputUri: taskAction.Spec.InputURI,
+ Group: taskAction.Spec.Group,
+ }
+ if taskAction.Spec.TaskType != "" {
+ ta := &workflow.TaskAction{
+ Id: &task.TaskIdentifier{
+ Project: taskAction.Spec.Project,
+ Domain: taskAction.Spec.Domain,
+ },
+ }
+ // Deserialize TaskTemplate to build TaskSpec
+ if len(taskAction.Spec.TaskTemplate) > 0 {
+ var tmpl core.TaskTemplate
+ if err := proto.Unmarshal(taskAction.Spec.TaskTemplate, &tmpl); err == nil {
+ if tmplID := tmpl.GetId(); tmplID != nil {
+ ta.Id.Name = tmplID.GetName()
+ ta.Id.Version = tmplID.GetVersion()
+ }
+ ta.Spec = &task.TaskSpec{
+ TaskTemplate: &tmpl,
+ ShortName: taskAction.Spec.ShortName,
+ }
+ }
+ }
+ recordReq.Spec = &workflow.RecordActionRequest_Task{
+ Task: ta,
+ }
+ }
+ if _, err := c.runClient.RecordAction(ctx, connect.NewRequest(recordReq)); err != nil {
+ logger.Warnf(ctx, "Failed to record action in run service for %s: %v", update.ActionID.Name, err)
+ } else if c.recordedFilter != nil {
+ c.recordedFilter.Add(ctx, actionKey)
+ }
+ }
+
+ // When a child action appears, the parent must already be running (it
+ // created the child). Promote the parent to RUNNING so the UI doesn't
+ // stay stuck on INITIALIZING while children are executing.
+ if !isDuplicate && update.ParentActionName != "" {
+ parentID := &common.ActionIdentifier{
+ Run: update.ActionID.Run,
+ Name: update.ParentActionName,
+ }
+ parentStatusReq := &workflow.UpdateActionStatusRequest{
+ ActionId: parentID,
+ Status: &workflow.ActionStatus{
+ Phase: common.ActionPhase_ACTION_PHASE_RUNNING,
+ },
+ }
+ if _, err := c.runClient.UpdateActionStatus(ctx, connect.NewRequest(parentStatusReq)); err != nil {
+ logger.Warnf(ctx, "Failed to promote parent action %s to RUNNING: %v", update.ParentActionName, err)
+ }
+ }
+ }
+
+ if update.Phase != common.ActionPhase_ACTION_PHASE_UNSPECIFIED {
+ statusReq := &workflow.UpdateActionStatusRequest{
+ ActionId: update.ActionID,
+ Status: &workflow.ActionStatus{
+ Phase: update.Phase,
+ Attempts: taskAction.Status.Attempts,
+ CacheStatus: taskAction.Status.CacheStatus,
+ },
+ }
+ if _, err := c.runClient.UpdateActionStatus(ctx, connect.NewRequest(statusReq)); err != nil {
+ logger.Warnf(ctx, "Failed to update action status in run service for %s: %v", update.ActionID.Name, err)
+ } else if isTerminalPhase(update.Phase) && !update.IsDeleted {
+ // Skip label patching for deleted CRs — the patch would always fail
+ // with "not found" since the object is already gone.
+ if err := c.markTerminalStatusRecorded(ctx, taskAction); err != nil {
+ logger.Warnf(ctx, "Failed to mark terminal status recorded for %s: %v", update.ActionID.Name, err)
+ }
+ }
+ }
+}
+
+// StopWatching stops the TaskAction watcher
+func (c *ActionsClient) StopWatching() {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ if c.watching && c.stopCh != nil {
+ close(c.stopCh)
+ c.watching = false
+ }
+}
+
+// GetPhaseFromConditions extracts the phase from TaskAction conditions.
+func GetPhaseFromConditions(taskAction *executorv1.TaskAction) common.ActionPhase {
+ for _, cond := range taskAction.Status.Conditions {
+ switch cond.Type {
+ case string(executorv1.ConditionTypeSucceeded):
+ if cond.Status == "True" {
+ return common.ActionPhase_ACTION_PHASE_SUCCEEDED
+ }
+ case string(executorv1.ConditionTypeFailed):
+ if cond.Status == "True" {
+ return common.ActionPhase_ACTION_PHASE_FAILED
+ }
+ case string(executorv1.ConditionTypeProgressing):
+ if cond.Status == "True" {
+ switch cond.Reason {
+ case string(executorv1.ConditionReasonQueued):
+ return common.ActionPhase_ACTION_PHASE_QUEUED
+ case string(executorv1.ConditionReasonInitializing):
+ return common.ActionPhase_ACTION_PHASE_INITIALIZING
+ case string(executorv1.ConditionReasonExecuting):
+ return common.ActionPhase_ACTION_PHASE_RUNNING
+ }
+ }
+ }
+ }
+ return common.ActionPhase_ACTION_PHASE_UNSPECIFIED
+}
+
+func isTerminalPhase(phase common.ActionPhase) bool {
+ return phase == common.ActionPhase_ACTION_PHASE_SUCCEEDED ||
+ phase == common.ActionPhase_ACTION_PHASE_FAILED ||
+ phase == common.ActionPhase_ACTION_PHASE_ABORTED ||
+ phase == common.ActionPhase_ACTION_PHASE_TIMED_OUT
+}
+
+func (c *ActionsClient) markTerminalStatusRecorded(ctx context.Context, taskAction *executorv1.TaskAction) error {
+ if c.k8sClient == nil || taskAction == nil || c.shouldSkipTaskAction(taskAction) {
+ return nil
+ }
+
+ original := taskAction.DeepCopy()
+ patched := taskAction.DeepCopy()
+ labels := patched.GetLabels()
+ if labels == nil {
+ labels = make(map[string]string)
+ }
+ labels[labelTerminalStatusRecorded] = "true"
+ patched.SetLabels(labels)
+
+ if err := c.k8sClient.Patch(ctx, patched, client.MergeFrom(original)); err != nil {
+ return err
+ }
+ return nil
+}
+
+// buildTaskActionName generates a Kubernetes-compliant name for the TaskAction.
+// For root actions (where action name == run name), the name is -a0.
+// For child actions, the name is -.
+func buildTaskActionName(actionID *common.ActionIdentifier) string {
+ isRoot := actionID.Name == actionID.Run.Name
+ if isRoot {
+ return fmt.Sprintf("%s-a0", actionID.Run.Name)
+ }
+ return fmt.Sprintf("%s-%s", actionID.Run.Name, actionID.Name)
+}
+
+// buildNamespace returns the Kubernetes namespace for a run: "-".
+func buildNamespace(runID *common.RunIdentifier) string {
+ return fmt.Sprintf("%s-%s", runID.Project, runID.Domain)
+}
+
+// buildOutputUri computes the action-specific output URI from the TaskAction spec.
+// It uses the same path structure as the executor's ComputeActionOutputPath so that
+// the SDK can find outputs written by the executor.
+func buildOutputUri(ctx context.Context, ta *executorv1.TaskAction) string {
+ if ta.Spec.RunOutputBase == "" {
+ return ""
+ }
+ attempt := ta.Status.Attempts
+ if attempt == 0 { // if attempts is not set, default to 1
+ attempt = 1
+ }
+ prefix, err := plugin.ComputeActionOutputPath(ctx, ta.Namespace, ta.Name, ta.Spec.RunOutputBase, ta.Spec.ActionName, attempt)
+ if err != nil {
+ return ""
+ }
+ return string(prefix)
+}
+
+// InitScheme adds the executor API types to the scheme
+func InitScheme() error {
+ return executorv1.AddToScheme(scheme.Scheme)
+}
+
+// buildActionSpec converts an actions.Action into the workflow.ActionSpec expected by the executor.
+func buildActionSpec(action *actions.Action, runSpec *task.RunSpec) *workflow.ActionSpec {
+ actionSpec := &workflow.ActionSpec{
+ ActionId: action.ActionId,
+ ParentActionName: action.ParentActionName,
+ RunSpec: runSpec,
+ InputUri: action.InputUri,
+ RunOutputBase: action.RunOutputBase,
+ Group: action.Group,
+ }
+
+ switch spec := action.Spec.(type) {
+ case *actions.Action_Task:
+ actionSpec.Spec = &workflow.ActionSpec_Task{Task: spec.Task}
+ case *actions.Action_Trace:
+ actionSpec.Spec = &workflow.ActionSpec_Trace{Trace: spec.Trace}
+ case *actions.Action_Condition:
+ actionSpec.Spec = &workflow.ActionSpec_Condition{Condition: spec.Condition}
+ }
+
+ return actionSpec
+}
+
+func applyRunSpecToTaskAction(taskAction *executorv1.TaskAction, runSpec *task.RunSpec) {
+ if runSpec == nil {
+ taskAction.Spec.EnvVars = nil
+ taskAction.Spec.Interruptible = nil
+ return
+ }
+
+ taskAction.Spec.EnvVars = keyValuePairsToMap(runSpec.GetEnvs().GetValues())
+ if runSpec.GetInterruptible() != nil {
+ value := runSpec.GetInterruptible().GetValue()
+ taskAction.Spec.Interruptible = &value
+ } else {
+ taskAction.Spec.Interruptible = nil
+ }
+
+ for key, value := range runSpec.GetLabels().GetValues() {
+ if _, exists := taskAction.Labels[key]; !exists {
+ taskAction.Labels[key] = value
+ }
+ }
+
+ if len(runSpec.GetAnnotations().GetValues()) > 0 {
+ if taskAction.Annotations == nil {
+ taskAction.Annotations = make(map[string]string, len(runSpec.GetAnnotations().GetValues()))
+ }
+ for key, value := range runSpec.GetAnnotations().GetValues() {
+ taskAction.Annotations[key] = value
+ }
+ }
+}
+
+func inheritRunContextFromParentTaskAction(taskAction *executorv1.TaskAction, parentTaskAction *executorv1.TaskAction) {
+ if taskAction == nil || parentTaskAction == nil {
+ return
+ }
+ taskAction.Spec.EnvVars = cloneStringMap(parentTaskAction.Spec.EnvVars)
+ if len(parentTaskAction.Annotations) > 0 {
+ if taskAction.Annotations == nil {
+ taskAction.Annotations = map[string]string{}
+ }
+ for k, v := range cloneStringMap(parentTaskAction.Annotations) {
+ if _, exists := taskAction.Annotations[k]; !exists {
+ taskAction.Annotations[k] = v
+ }
+ }
+ }
+ if len(parentTaskAction.Labels) > 0 {
+ if taskAction.Labels == nil {
+ taskAction.Labels = map[string]string{}
+ }
+ for k, v := range cloneStringMap(parentTaskAction.Labels) {
+ if _, exists := taskAction.Labels[k]; !exists {
+ taskAction.Labels[k] = v
+ }
+ }
+ }
+ if parentTaskAction.Spec.Interruptible != nil {
+ v := *parentTaskAction.Spec.Interruptible
+ taskAction.Spec.Interruptible = &v
+ } else {
+ taskAction.Spec.Interruptible = nil
+ }
+}
+
+func cloneStringMap(src map[string]string) map[string]string {
+ if len(src) == 0 {
+ return nil
+ }
+ out := make(map[string]string, len(src))
+ for k, v := range src {
+ out[k] = v
+ }
+ return out
+}
+
+func keyValuePairsToMap(values []*core.KeyValuePair) map[string]string {
+ if len(values) == 0 {
+ return nil
+ }
+
+ cloned := make(map[string]string, len(values))
+ for _, kv := range values {
+ if kv == nil {
+ continue
+ }
+ cloned[kv.GetKey()] = kv.GetValue()
+ }
+ return cloned
+}
+
+func extractTaskCacheKey(action *actions.Action) string {
+ taskSpec, ok := action.Spec.(*actions.Action_Task)
+ if !ok || taskSpec.Task == nil {
+ return ""
+ }
+
+ if taskSpec.Task.CacheKey == nil {
+ return ""
+ }
+
+ return taskSpec.Task.CacheKey.Value
+}
+
+// embedTaskTemplate serializes the inline TaskTemplate from the Action into the CR spec.
+func embedTaskTemplate(action *actions.Action, taskAction *executorv1.TaskAction) error {
+ taskSpec, ok := action.Spec.(*actions.Action_Task)
+ if !ok || taskSpec.Task == nil || taskSpec.Task.Spec == nil || taskSpec.Task.Spec.TaskTemplate == nil {
+ // Non-task actions do not carry an inline template.
+ return nil
+ }
+
+ tmpl := taskSpec.Task.Spec.TaskTemplate
+ taskAction.Spec.TaskType = tmpl.Type
+ taskAction.Spec.ShortName = taskSpec.Task.Spec.ShortName
+
+ data, err := proto.Marshal(tmpl)
+ if err != nil {
+ return fmt.Errorf("failed to marshal task template: %w", err)
+ }
+ taskAction.Spec.TaskTemplate = data
+ return nil
+}
+
+// extractShortNameFromTemplate extracts a human-readable function name from a serialized TaskTemplate.
+// It splits on '.' and returns the last part.
+func extractShortNameFromTemplate(templateBytes []byte) string {
+ tmpl := &core.TaskTemplate{}
+ if err := proto.Unmarshal(templateBytes, tmpl); err != nil {
+ return ""
+ }
+ if tmpl.GetId() == nil {
+ return ""
+ }
+ name := tmpl.GetId().GetName()
+ if name == "" {
+ return ""
+ }
+ // Split on '.' and take the last part
+ parts := strings.Split(name, ".")
+ if len(parts) > 0 {
+ return parts[len(parts)-1]
+ }
+ return name
+}
diff --git a/actions/k8s/client_test.go b/actions/k8s/client_test.go
new file mode 100644
index 00000000000..bd357dbd303
--- /dev/null
+++ b/actions/k8s/client_test.go
@@ -0,0 +1,720 @@
+package k8s
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "connectrpc.com/connect"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/types/known/wrapperspb"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/watch"
+
+ executorv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+ "github.com/flyteorg/flyte/v2/flytestdlib/fastcheck"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/actions"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/task"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow"
+ runmocks "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow/workflowconnect/mocks"
+)
+
+func newTestActionUpdate(actionName string) (*executorv1.TaskAction, *ActionUpdate) {
+ runID := &common.RunIdentifier{
+ Project: "proj",
+ Domain: "dev",
+ Name: "run1",
+ }
+ ta := &executorv1.TaskAction{
+ Spec: executorv1.TaskActionSpec{
+ Project: runID.Project,
+ Domain: runID.Domain,
+ RunName: runID.Name,
+ ActionName: actionName,
+ },
+ }
+ update := &ActionUpdate{
+ ActionID: &common.ActionIdentifier{
+ Run: runID,
+ Name: actionName,
+ },
+ }
+ return ta, update
+}
+
+func TestNotifyRunService_DeduplicateRecordAction(t *testing.T) {
+ ctx := context.Background()
+
+ mockClient := runmocks.NewInternalRunServiceClient(t)
+
+ filter, err := fastcheck.NewOppoBloomFilter(128, promutils.NewTestScope())
+ assert.NoError(t, err)
+
+ c := &ActionsClient{
+ runClient: mockClient,
+ recordedFilter: filter,
+ subscribers: make(map[string]map[chan *ActionUpdate]struct{}),
+ }
+
+ ta, update := newTestActionUpdate("action-1")
+
+ // Expect RecordAction called exactly once
+ mockClient.On("RecordAction", mock.Anything, mock.Anything).
+ Return(&connect.Response[workflow.RecordActionResponse]{}, nil).Once()
+
+ // First Added event — should call RecordAction
+ c.notifyRunService(ctx, ta, update, watch.Added)
+
+ // Second Added event (replay) — should be deduplicated
+ c.notifyRunService(ctx, ta, update, watch.Added)
+
+ mockClient.AssertNumberOfCalls(t, "RecordAction", 1)
+}
+
+func TestNotifyRunService_FailedRecordAllowsRetry(t *testing.T) {
+ ctx := context.Background()
+
+ mockClient := runmocks.NewInternalRunServiceClient(t)
+
+ filter, err := fastcheck.NewOppoBloomFilter(128, promutils.NewTestScope())
+ assert.NoError(t, err)
+
+ c := &ActionsClient{
+ runClient: mockClient,
+ recordedFilter: filter,
+ subscribers: make(map[string]map[chan *ActionUpdate]struct{}),
+ }
+
+ ta, update := newTestActionUpdate("action-2")
+
+ // First call fails
+ mockClient.On("RecordAction", mock.Anything, mock.Anything).
+ Return((*connect.Response[workflow.RecordActionResponse])(nil), fmt.Errorf("transient error")).Once()
+ // Second call succeeds
+ mockClient.On("RecordAction", mock.Anything, mock.Anything).
+ Return(&connect.Response[workflow.RecordActionResponse]{}, nil).Once()
+
+ // First event — RecordAction fails, should NOT add to filter
+ c.notifyRunService(ctx, ta, update, watch.Added)
+
+ // Second event — should retry RecordAction since first failed
+ c.notifyRunService(ctx, ta, update, watch.Added)
+
+ mockClient.AssertNumberOfCalls(t, "RecordAction", 2)
+
+ // Third event — now it's in the filter, should be skipped
+ c.notifyRunService(ctx, ta, update, watch.Added)
+ mockClient.AssertNumberOfCalls(t, "RecordAction", 2)
+}
+
+func TestNotifyRunService_NilFilter(t *testing.T) {
+ ctx := context.Background()
+
+ mockClient := runmocks.NewInternalRunServiceClient(t)
+
+ // No filter — should always call RecordAction
+ c := &ActionsClient{
+ runClient: mockClient,
+ subscribers: make(map[string]map[chan *ActionUpdate]struct{}),
+ }
+
+ ta, update := newTestActionUpdate("action-3")
+
+ mockClient.On("RecordAction", mock.Anything, mock.Anything).
+ Return(&connect.Response[workflow.RecordActionResponse]{}, nil)
+
+ c.notifyRunService(ctx, ta, update, watch.Added)
+ c.notifyRunService(ctx, ta, update, watch.Added)
+
+ mockClient.AssertNumberOfCalls(t, "RecordAction", 2)
+}
+
+func TestNotifyRunService_UpdateActionStatusIncludesAttemptsAndCacheStatus(t *testing.T) {
+ ctx := context.Background()
+
+ mockClient := runmocks.NewInternalRunServiceClient(t)
+ c := &ActionsClient{
+ runClient: mockClient,
+ subscribers: make(map[string]map[chan *ActionUpdate]struct{}),
+ }
+
+ ta, update := newTestActionUpdate("action-4")
+ ta.Status.Attempts = 3
+ ta.Status.CacheStatus = core.CatalogCacheStatus_CACHE_HIT
+ update.Phase = common.ActionPhase_ACTION_PHASE_SUCCEEDED
+
+ mockClient.On("UpdateActionStatus", mock.Anything, mock.MatchedBy(func(req *connect.Request[workflow.UpdateActionStatusRequest]) bool {
+ status := req.Msg.GetStatus()
+ return status.GetPhase() == common.ActionPhase_ACTION_PHASE_SUCCEEDED &&
+ status.GetAttempts() == 3 &&
+ status.GetCacheStatus() == core.CatalogCacheStatus_CACHE_HIT
+ })).Return(&connect.Response[workflow.UpdateActionStatusResponse]{}, nil).Once()
+
+ c.notifyRunService(ctx, ta, update, watch.Modified)
+
+ mockClient.AssertNumberOfCalls(t, "UpdateActionStatus", 1)
+}
+
+func TestBuildTaskActionName(t *testing.T) {
+ runID := &common.RunIdentifier{
+ Project: "project",
+ Domain: "development",
+ Name: "rabc123",
+ }
+
+ t.Run("root action uses a0 suffix", func(t *testing.T) {
+ // Root: action name == run name
+ actionID := &common.ActionIdentifier{
+ Run: runID,
+ Name: runID.Name,
+ }
+ assert.Equal(t, "rabc123-a0", buildTaskActionName(actionID))
+ })
+
+ t.Run("child action includes action name", func(t *testing.T) {
+ actionID := &common.ActionIdentifier{
+ Run: runID,
+ Name: "train",
+ }
+ assert.Equal(t, "rabc123-train", buildTaskActionName(actionID))
+ })
+}
+
+func TestBuildNamespace(t *testing.T) {
+ t.Run("combines project and domain", func(t *testing.T) {
+ runID := &common.RunIdentifier{
+ Project: "flytesnacks",
+ Domain: "development",
+ }
+ assert.Equal(t, "flytesnacks-development", buildNamespace(runID))
+ })
+
+ t.Run("different project and domain", func(t *testing.T) {
+ runID := &common.RunIdentifier{
+ Project: "myproject",
+ Domain: "production",
+ }
+ assert.Equal(t, "myproject-production", buildNamespace(runID))
+ })
+}
+
+func TestExtractTaskCacheKey(t *testing.T) {
+ t.Run("returns cache key for task action", func(t *testing.T) {
+ action := &actions.Action{
+ Spec: &actions.Action_Task{
+ Task: &workflow.TaskAction{
+ CacheKey: wrapperspb.String("cache-v1"),
+ },
+ },
+ }
+
+ assert.Equal(t, "cache-v1", extractTaskCacheKey(action))
+ })
+
+ t.Run("returns empty for non-task action", func(t *testing.T) {
+ assert.Empty(t, extractTaskCacheKey(&actions.Action{}))
+ })
+}
+
+func TestApplyRunSpecToTaskAction_ProjectsRuntimeSettings(t *testing.T) {
+ taskAction := &executorv1.TaskAction{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: map[string]string{
+ "flyte.org/run": "run1",
+ },
+ },
+ Spec: executorv1.TaskActionSpec{},
+ }
+
+ applyRunSpecToTaskAction(taskAction, &task.RunSpec{
+ Envs: &task.Envs{
+ Values: []*core.KeyValuePair{
+ {Key: "TRACE_ID", Value: "abc123"},
+ },
+ },
+ Interruptible: wrapperspb.Bool(true),
+ Labels: &task.Labels{
+ Values: map[string]string{
+ "flyte.org/run": "should-not-override",
+ "team": "platform",
+ },
+ },
+ Annotations: &task.Annotations{
+ Values: map[string]string{
+ "owner": "sdk",
+ },
+ },
+ })
+
+ require.NotNil(t, taskAction.Spec.Interruptible)
+ assert.True(t, *taskAction.Spec.Interruptible)
+ assert.Len(t, taskAction.Spec.EnvVars, 1)
+ assert.Equal(t, "abc123", taskAction.Spec.EnvVars["TRACE_ID"])
+ assert.Equal(t, "run1", taskAction.Labels["flyte.org/run"])
+ assert.Equal(t, "platform", taskAction.Labels["team"])
+ assert.Equal(t, "sdk", taskAction.Annotations["owner"])
+}
+
+func TestInheritRunContextFromParentTaskAction(t *testing.T) {
+ interruptible := true
+ parent := &executorv1.TaskAction{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: map[string]string{
+ "team": "platform",
+ },
+ Annotations: map[string]string{
+ "owner": "sdk",
+ },
+ },
+ Spec: executorv1.TaskActionSpec{
+ EnvVars: map[string]string{
+ "TRACE_ID": "abc123",
+ "TEAM": "platform",
+ },
+ Interruptible: &interruptible,
+ },
+ }
+
+ child := &executorv1.TaskAction{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: map[string]string{
+ "flyte.org/run": "run1",
+ },
+ },
+ Spec: executorv1.TaskActionSpec{},
+ }
+
+ inheritRunContextFromParentTaskAction(child, parent)
+
+ require.NotNil(t, child.Spec.Interruptible)
+ assert.True(t, *child.Spec.Interruptible)
+ assert.Equal(t, parent.Spec.EnvVars, child.Spec.EnvVars)
+ assert.Equal(t, "platform", child.Labels["team"])
+ assert.Equal(t, "run1", child.Labels["flyte.org/run"])
+ assert.Equal(t, "sdk", child.Annotations["owner"])
+
+ // Verify deep copy (child mutation must not mutate parent map).
+ child.Spec.EnvVars["TRACE_ID"] = "mutated"
+ assert.Equal(t, "abc123", parent.Spec.EnvVars["TRACE_ID"])
+ child.Annotations["owner"] = "mutated"
+ assert.Equal(t, "sdk", parent.Annotations["owner"])
+}
+
+func TestInheritRunContextFromParentTaskAction_DoesNotOverrideExistingLabels(t *testing.T) {
+ parentInterruptible := true
+ parent := &executorv1.TaskAction{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: map[string]string{
+ "flyte.org/run": "should-not-override",
+ "team": "platform",
+ },
+ },
+ Spec: executorv1.TaskActionSpec{
+ EnvVars: map[string]string{
+ "TRACE_ID": "parent",
+ "TEAM": "platform",
+ },
+ Interruptible: &parentInterruptible,
+ },
+ }
+
+ child := &executorv1.TaskAction{
+ ObjectMeta: metav1.ObjectMeta{
+ Labels: map[string]string{
+ "flyte.org/run": "run1",
+ },
+ },
+ Spec: executorv1.TaskActionSpec{},
+ }
+
+ inheritRunContextFromParentTaskAction(child, parent)
+
+ require.NotNil(t, child.Spec.Interruptible)
+ assert.True(t, *child.Spec.Interruptible)
+ assert.Equal(t, map[string]string{"TRACE_ID": "parent", "TEAM": "platform"}, child.Spec.EnvVars)
+ assert.Equal(t, "run1", child.Labels["flyte.org/run"])
+ assert.Equal(t, "platform", child.Labels["team"])
+}
+
+func TestNotifyRunService_ChildAddedPromotesParentToRunning(t *testing.T) {
+ ctx := context.Background()
+
+ mockClient := runmocks.NewInternalRunServiceClient(t)
+ c := &ActionsClient{
+ runClient: mockClient,
+ subscribers: make(map[string]map[chan *ActionUpdate]struct{}),
+ }
+
+ runID := &common.RunIdentifier{
+ Project: "proj",
+ Domain: "dev",
+ Name: "run1",
+ }
+
+ ta := &executorv1.TaskAction{
+ Spec: executorv1.TaskActionSpec{
+ Project: runID.Project,
+ Domain: runID.Domain,
+ RunName: runID.Name,
+ ActionName: "child-1",
+ },
+ }
+ update := &ActionUpdate{
+ ActionID: &common.ActionIdentifier{
+ Run: runID,
+ Name: "child-1",
+ },
+ ParentActionName: "run1",
+ }
+
+ // Expect RecordAction for the child
+ mockClient.On("RecordAction", mock.Anything, mock.Anything).
+ Return(&connect.Response[workflow.RecordActionResponse]{}, nil).Once()
+
+ // Expect UpdateActionStatus for the PARENT with RUNNING phase
+ mockClient.On("UpdateActionStatus", mock.Anything, mock.MatchedBy(func(req *connect.Request[workflow.UpdateActionStatusRequest]) bool {
+ return req.Msg.GetActionId().GetName() == "run1" &&
+ req.Msg.GetStatus().GetPhase() == common.ActionPhase_ACTION_PHASE_RUNNING
+ })).Return(&connect.Response[workflow.UpdateActionStatusResponse]{}, nil).Once()
+
+ c.notifyRunService(ctx, ta, update, watch.Added)
+
+ mockClient.AssertNumberOfCalls(t, "RecordAction", 1)
+ mockClient.AssertNumberOfCalls(t, "UpdateActionStatus", 1)
+}
+
+func TestNotifyRunService_SkipsTerminalAddedEventsOnlyWhenInBloomFilter(t *testing.T) {
+ ctx := context.Background()
+
+ mockClient := runmocks.NewInternalRunServiceClient(t)
+ filter, err := fastcheck.NewOppoBloomFilter(128, promutils.NewTestScope())
+ require.NoError(t, err)
+
+ c := &ActionsClient{
+ runClient: mockClient,
+ recordedFilter: filter,
+ subscribers: make(map[string]map[chan *ActionUpdate]struct{}),
+ }
+
+ // Create a terminal TaskAction CRD (SUCCEEDED)
+ ta := &executorv1.TaskAction{
+ Spec: executorv1.TaskActionSpec{
+ Project: "proj",
+ Domain: "dev",
+ RunName: "run1",
+ ActionName: "completed-action",
+ },
+ Status: executorv1.TaskActionStatus{
+ Conditions: []metav1.Condition{
+ {Type: string(executorv1.ConditionTypeSucceeded), Status: "True"},
+ },
+ },
+ }
+ update := &ActionUpdate{
+ ActionID: &common.ActionIdentifier{
+ Run: &common.RunIdentifier{Project: "proj", Domain: "dev", Name: "run1"},
+ Name: "completed-action",
+ },
+ Phase: common.ActionPhase_ACTION_PHASE_SUCCEEDED,
+ }
+
+ // First ADDED event (cold start, not in bloom filter): should process normally
+ mockClient.On("RecordAction", mock.Anything, mock.Anything).
+ Return(&connect.Response[workflow.RecordActionResponse]{}, nil).Once()
+ mockClient.On("UpdateActionStatus", mock.Anything, mock.Anything).
+ Return(&connect.Response[workflow.UpdateActionStatusResponse]{}, nil)
+ c.notifyRunService(ctx, ta, update, watch.Added)
+
+ mockClient.AssertNumberOfCalls(t, "RecordAction", 1)
+ mockClient.AssertNumberOfCalls(t, "UpdateActionStatus", 1)
+
+ // Action should now be in the bloom filter
+ actionKey := []byte(buildTaskActionName(update.ActionID))
+ assert.True(t, filter.Contains(ctx, actionKey))
+
+ // Second ADDED event (reconnect, in bloom filter): should skip RecordAction
+ // but still call UpdateActionStatus.
+ c.notifyRunService(ctx, ta, update, watch.Added)
+ mockClient.AssertNumberOfCalls(t, "RecordAction", 1) // no new RecordAction
+ mockClient.AssertNumberOfCalls(t, "UpdateActionStatus", 2) // one more UpdateActionStatus
+}
+
+func TestNotifyRunService_ProcessesNonTerminalAddedEvents(t *testing.T) {
+ ctx := context.Background()
+
+ mockClient := runmocks.NewInternalRunServiceClient(t)
+ filter, err := fastcheck.NewOppoBloomFilter(128, promutils.NewTestScope())
+ require.NoError(t, err)
+
+ c := &ActionsClient{
+ runClient: mockClient,
+ recordedFilter: filter,
+ subscribers: make(map[string]map[chan *ActionUpdate]struct{}),
+ }
+
+ // Create a non-terminal TaskAction CRD (QUEUED)
+ ta := &executorv1.TaskAction{
+ Spec: executorv1.TaskActionSpec{
+ Project: "proj",
+ Domain: "dev",
+ RunName: "run1",
+ ActionName: "queued-action",
+ },
+ Status: executorv1.TaskActionStatus{
+ Conditions: []metav1.Condition{
+ {Type: string(executorv1.ConditionTypeProgressing), Status: "True", Reason: string(executorv1.ConditionReasonQueued)},
+ },
+ },
+ }
+ update := &ActionUpdate{
+ ActionID: &common.ActionIdentifier{
+ Run: &common.RunIdentifier{Project: "proj", Domain: "dev", Name: "run1"},
+ Name: "queued-action",
+ },
+ Phase: common.ActionPhase_ACTION_PHASE_QUEUED,
+ }
+
+ // Non-terminal ADDED events should be processed normally
+ mockClient.On("RecordAction", mock.Anything, mock.Anything).
+ Return(&connect.Response[workflow.RecordActionResponse]{}, nil).Once()
+ mockClient.On("UpdateActionStatus", mock.Anything, mock.Anything).
+ Return(&connect.Response[workflow.UpdateActionStatusResponse]{}, nil).Once()
+
+ c.notifyRunService(ctx, ta, update, watch.Added)
+
+ mockClient.AssertNumberOfCalls(t, "RecordAction", 1)
+ mockClient.AssertNumberOfCalls(t, "UpdateActionStatus", 1)
+}
+
+func TestNotifyRunService_DuplicateAddedSkipsRecordAction(t *testing.T) {
+ ctx := context.Background()
+
+ mockClient := runmocks.NewInternalRunServiceClient(t)
+ filter, err := fastcheck.NewOppoBloomFilter(128, promutils.NewTestScope())
+ require.NoError(t, err)
+
+ c := &ActionsClient{
+ runClient: mockClient,
+ recordedFilter: filter,
+ subscribers: make(map[string]map[chan *ActionUpdate]struct{}),
+ }
+
+ ta, update := newTestActionUpdate("action-dup")
+ update.Phase = common.ActionPhase_ACTION_PHASE_RUNNING
+
+ // First call — should process normally
+ mockClient.On("RecordAction", mock.Anything, mock.Anything).
+ Return(&connect.Response[workflow.RecordActionResponse]{}, nil).Once()
+ mockClient.On("UpdateActionStatus", mock.Anything, mock.Anything).
+ Return(&connect.Response[workflow.UpdateActionStatusResponse]{}, nil)
+ c.notifyRunService(ctx, ta, update, watch.Added)
+
+ // Second call (duplicate ADDED) — should skip RecordAction but still call UpdateActionStatus
+ c.notifyRunService(ctx, ta, update, watch.Added)
+
+ mockClient.AssertNumberOfCalls(t, "RecordAction", 1)
+ mockClient.AssertNumberOfCalls(t, "UpdateActionStatus", 2)
+}
+
+func TestNotifyRunService_TerminalDuplicateRepairsTimestamps(t *testing.T) {
+ ctx := context.Background()
+
+ mockClient := runmocks.NewInternalRunServiceClient(t)
+ filter, err := fastcheck.NewOppoBloomFilter(128, promutils.NewTestScope())
+ require.NoError(t, err)
+
+ c := &ActionsClient{
+ runClient: mockClient,
+ recordedFilter: filter,
+ subscribers: make(map[string]map[chan *ActionUpdate]struct{}),
+ }
+
+ ta, update := newTestActionUpdate("action-terminal-dup")
+ update.Phase = common.ActionPhase_ACTION_PHASE_SUCCEEDED
+
+ // First call — should process normally (RecordAction + UpdateActionStatus)
+ mockClient.On("RecordAction", mock.Anything, mock.Anything).
+ Return(&connect.Response[workflow.RecordActionResponse]{}, nil).Once()
+ mockClient.On("UpdateActionStatus", mock.Anything, mock.Anything).
+ Return(&connect.Response[workflow.UpdateActionStatusResponse]{}, nil).Times(2)
+ c.notifyRunService(ctx, ta, update, watch.Added)
+
+ // Second call (terminal duplicate ADDED) — should skip RecordAction but
+ // still call UpdateActionStatus to repair missing timestamps.
+ c.notifyRunService(ctx, ta, update, watch.Added)
+
+ mockClient.AssertNumberOfCalls(t, "RecordAction", 1) // no new RecordAction
+ mockClient.AssertNumberOfCalls(t, "UpdateActionStatus", 2) // one more UpdateActionStatus
+}
+
+func TestNotifyRunService_RootActionAddedDoesNotPromoteParent(t *testing.T) {
+ ctx := context.Background()
+
+ mockClient := runmocks.NewInternalRunServiceClient(t)
+ c := &ActionsClient{
+ runClient: mockClient,
+ subscribers: make(map[string]map[chan *ActionUpdate]struct{}),
+ }
+
+ // Root action has no parent
+ ta, update := newTestActionUpdate("action-root")
+
+ mockClient.On("RecordAction", mock.Anything, mock.Anything).
+ Return(&connect.Response[workflow.RecordActionResponse]{}, nil).Once()
+
+ c.notifyRunService(ctx, ta, update, watch.Added)
+
+ mockClient.AssertNumberOfCalls(t, "RecordAction", 1)
+ // No UpdateActionStatus should be called for root (no parent to promote)
+ mockClient.AssertNumberOfCalls(t, "UpdateActionStatus", 0)
+}
+
+func newWorkerTestClient(numWorkers, bufSize int) *ActionsClient {
+ c := &ActionsClient{
+ numWorkers: numWorkers,
+ workerChs: make([]chan watch.Event, numWorkers),
+ }
+ for i := range c.workerChs {
+ c.workerChs[i] = make(chan watch.Event, bufSize)
+ }
+ return c
+}
+
+func newTaskActionEvent(name string, eventType watch.EventType) watch.Event {
+ ta := &executorv1.TaskAction{}
+ ta.Name = name
+ return watch.Event{Type: eventType, Object: ta}
+}
+
+func newTaskAction(name string) *executorv1.TaskAction {
+ ta := &executorv1.TaskAction{}
+ ta.Name = name
+ return ta
+}
+
+func TestDispatchEvent_ConsistentSharding(t *testing.T) {
+ c := newWorkerTestClient(4, 10)
+ taskAction := newTaskAction("run1-action1")
+
+ c.dispatchEvent(taskAction, watch.Modified)
+ c.dispatchEvent(taskAction, watch.Modified)
+
+ // Both events must land in exactly one shard (the same one each time)
+ var total int
+ for _, ch := range c.workerChs {
+ total += len(ch)
+ }
+ assert.Equal(t, 2, total)
+}
+
+func TestDispatchEvent_DifferentNamesCanLandOnDifferentShards(t *testing.T) {
+ c := newWorkerTestClient(4, 10)
+
+ // Send many differently-named events and confirm they spread across shards
+ names := []string{"run1-a", "run1-b", "run1-c", "run1-d", "run1-e", "run1-f", "run1-g", "run1-h"}
+ for _, name := range names {
+ c.dispatchEvent(newTaskAction(name), watch.Modified)
+ }
+
+ var nonEmpty int
+ for _, ch := range c.workerChs {
+ if len(ch) > 0 {
+ nonEmpty++
+ }
+ }
+ assert.Greater(t, nonEmpty, 1, "expected events to spread across more than one shard")
+}
+
+func TestDispatchEvent_NilTaskActionIsIgnored(t *testing.T) {
+ c := newWorkerTestClient(2, 10)
+
+ c.dispatchEvent(nil, watch.Modified)
+
+ for i, ch := range c.workerChs {
+ assert.Equal(t, 0, len(ch), "worker %d should have received nothing", i)
+ }
+}
+
+func TestDispatchEvent_FullChannelBlocks(t *testing.T) {
+ c := newWorkerTestClient(1, 1) // single worker, capacity 1
+ taskAction := newTaskAction("run1-action1")
+
+ c.dispatchEvent(taskAction, watch.Modified) // fills the channel
+
+ // Second dispatch must block because the channel is full.
+ // Run it in a goroutine and verify it unblocks once the channel is drained.
+ done := make(chan struct{})
+ go func() {
+ c.dispatchEvent(taskAction, watch.Modified)
+ close(done)
+ }()
+
+ // Drain the channel to unblock the sender.
+ <-c.workerChs[0]
+
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ t.Fatal("dispatchEvent did not unblock after channel was drained")
+ }
+}
+
+func TestDispatchEvent_DeepCopiesTaskAction(t *testing.T) {
+ c := newWorkerTestClient(1, 10)
+ taskAction := newTaskAction("run1-action1")
+
+ c.dispatchEvent(taskAction, watch.Modified)
+ taskAction.Name = "mutated-after-dispatch"
+
+ event := <-c.workerChs[0]
+ dispatched, ok := event.Object.(*executorv1.TaskAction)
+ require.True(t, ok)
+ assert.Equal(t, "run1-action1", dispatched.Name)
+}
+
+func TestWorker_ExitsOnStopCh(t *testing.T) {
+ c := &ActionsClient{}
+ ch := make(chan watch.Event, 10)
+ stopCh := make(chan struct{})
+
+ done := make(chan struct{})
+ go func() {
+ c.worker(context.Background(), ch, stopCh)
+ close(done)
+ }()
+
+ close(stopCh)
+
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ t.Fatal("worker did not exit after stopCh was closed")
+ }
+}
+
+func TestWorker_ExitsOnContextCancel(t *testing.T) {
+ c := &ActionsClient{}
+ ch := make(chan watch.Event, 10)
+ stopCh := make(chan struct{})
+ ctx, cancel := context.WithCancel(context.Background())
+
+ done := make(chan struct{})
+ go func() {
+ c.worker(ctx, ch, stopCh)
+ close(done)
+ }()
+
+ cancel()
+
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ t.Fatal("worker did not exit after context was cancelled")
+ }
+}
diff --git a/actions/service/actions_service.go b/actions/service/actions_service.go
new file mode 100644
index 00000000000..3a78b850fd2
--- /dev/null
+++ b/actions/service/actions_service.go
@@ -0,0 +1,244 @@
+package service
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+
+ "connectrpc.com/connect"
+
+ executorv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/actions"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/actions/actionsconnect"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow"
+)
+
+// ActionsService implements the ActionsService gRPC API.
+type ActionsService struct {
+ client ActionsClientInterface
+}
+
+func (s *ActionsService) Signal(ctx context.Context, c *connect.Request[actions.SignalRequest]) (*connect.Response[actions.SignalResponse], error) {
+ return nil, connect.NewError(connect.CodeUnimplemented, errors.New("endpoint Signal not implemented"))
+}
+
+// NewActionsService creates a new ActionsService.
+func NewActionsService(client ActionsClientInterface) *ActionsService {
+ return &ActionsService{client: client}
+}
+
+// Ensure we implement the interface.
+var _ actionsconnect.ActionsServiceHandler = (*ActionsService)(nil)
+
+// Enqueue queues a new action for execution.
+func (s *ActionsService) Enqueue(
+ ctx context.Context,
+ req *connect.Request[actions.EnqueueRequest],
+) (*connect.Response[actions.EnqueueResponse], error) {
+ logger.Infof(ctx, "ActionsService.Enqueue called")
+
+ if err := req.Msg.Validate(); err != nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, err)
+ }
+
+ if err := s.client.Enqueue(ctx, req.Msg.Action, req.Msg.RunSpec); err != nil {
+ logger.Errorf(ctx, "Failed to enqueue action: %v", err)
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+
+ return connect.NewResponse(&actions.EnqueueResponse{}), nil
+}
+
+// GetLatestState returns the latest state of an action.
+func (s *ActionsService) GetLatestState(
+ ctx context.Context,
+ req *connect.Request[actions.GetLatestStateRequest],
+) (*connect.Response[actions.GetLatestStateResponse], error) {
+ logger.Infof(ctx, "ActionsService.GetLatestState called")
+
+ if err := req.Msg.Validate(); err != nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, err)
+ }
+
+ state, err := s.client.GetState(ctx, req.Msg.ActionId)
+ if err != nil {
+ logger.Errorf(ctx, "Failed to get state: %v", err)
+ return nil, connect.NewError(connect.CodeNotFound, err)
+ }
+
+ return connect.NewResponse(&actions.GetLatestStateResponse{State: state}), nil
+}
+
+// WatchForUpdates watches for updates to the state of actions.
+func (s *ActionsService) WatchForUpdates(
+ ctx context.Context,
+ req *connect.Request[actions.WatchForUpdatesRequest],
+ stream *connect.ServerStream[actions.WatchForUpdatesResponse],
+) error {
+ logger.Infof(ctx, "ActionsService.WatchForUpdates stream started")
+
+ var parentActionID *common.ActionIdentifier
+ switch filter := req.Msg.Filter.(type) {
+ case *actions.WatchForUpdatesRequest_ParentActionId:
+ parentActionID = filter.ParentActionId
+ default:
+ return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("filter is required"))
+ }
+
+ if parentActionID == nil {
+ return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("parent_action_id is required"))
+ }
+
+ // Subscribe before listing to avoid missing events between snapshot and watch.
+ updateCh := s.client.Subscribe(parentActionID.Name)
+ defer s.client.Unsubscribe(parentActionID.Name, updateCh)
+
+ // Send initial state snapshot.
+ childActions, err := s.client.ListChildActions(ctx, parentActionID)
+ if err != nil {
+ return connect.NewError(connect.CodeInternal, fmt.Errorf("failed to list child actions: %w", err))
+ }
+
+ for _, action := range childActions {
+ resp := &actions.WatchForUpdatesResponse{
+ Message: &actions.WatchForUpdatesResponse_ActionUpdate{
+ ActionUpdate: taskActionToUpdate(action),
+ },
+ }
+ if err := stream.Send(resp); err != nil {
+ return err
+ }
+ }
+
+ // Send sentinel to signal end of initial snapshot.
+ sentinel := &actions.WatchForUpdatesResponse{
+ Message: &actions.WatchForUpdatesResponse_ControlMessage{
+ ControlMessage: &workflow.ControlMessage{Sentinel: true},
+ },
+ }
+ if err := stream.Send(sentinel); err != nil {
+ return err
+ }
+
+ logger.Infof(ctx, "Sent initial state (%d actions) and sentinel for parent: %s", len(childActions), parentActionID.Name)
+
+ for {
+ select {
+ case <-ctx.Done():
+ logger.Infof(ctx, "ActionsService.WatchForUpdates stream closed by client")
+ return nil
+
+ case update, ok := <-updateCh:
+ if !ok {
+ logger.Infof(ctx, "Update channel closed")
+ return nil
+ }
+
+ resp := &actions.WatchForUpdatesResponse{
+ Message: &actions.WatchForUpdatesResponse_ActionUpdate{
+ ActionUpdate: &workflow.ActionUpdate{
+ ActionId: update.ActionID,
+ Phase: update.Phase,
+ OutputUri: update.OutputUri,
+ },
+ },
+ }
+ if err := stream.Send(resp); err != nil {
+ return err
+ }
+
+ logger.Debugf(ctx, "Sent action update for: %s", update.ActionID.Name)
+ }
+ }
+}
+
+// Update updates the status of an action and saves its serialized state.
+func (s *ActionsService) Update(
+ ctx context.Context,
+ req *connect.Request[actions.UpdateRequest],
+) (*connect.Response[actions.UpdateResponse], error) {
+ logger.Infof(ctx, "ActionsService.Update called")
+
+ if err := req.Msg.Validate(); err != nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, err)
+ }
+
+ if err := s.client.PutState(ctx, req.Msg.ActionId, req.Msg.Attempt, req.Msg.Status, req.Msg.State); err != nil {
+ logger.Errorf(ctx, "Failed to update action: %v", err)
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+
+ return connect.NewResponse(&actions.UpdateResponse{}), nil
+}
+
+// Abort aborts a queued or running action, cascading to descendants.
+func (s *ActionsService) Abort(
+ ctx context.Context,
+ req *connect.Request[actions.AbortRequest],
+) (*connect.Response[actions.AbortResponse], error) {
+ logger.Infof(ctx, "ActionsService.Abort called")
+
+ if err := req.Msg.Validate(); err != nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, err)
+ }
+
+ if err := s.client.AbortAction(ctx, req.Msg.ActionId, req.Msg.Reason); err != nil {
+ logger.Errorf(ctx, "Failed to abort action: %v", err)
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+
+ return connect.NewResponse(&actions.AbortResponse{}), nil
+}
+
+// taskActionToUpdate converts a TaskAction CR to a workflow.ActionUpdate.
+func taskActionToUpdate(action *executorv1.TaskAction) *workflow.ActionUpdate {
+ return &workflow.ActionUpdate{
+ ActionId: &common.ActionIdentifier{
+ Run: &common.RunIdentifier{
+ Project: action.Spec.Project,
+ Domain: action.Spec.Domain,
+ Name: action.Spec.RunName,
+ },
+ Name: action.Spec.ActionName,
+ },
+ Phase: getPhaseFromConditions(action),
+ OutputUri: actionOutputURI(action.Spec.RunOutputBase, action.Spec.ActionName),
+ }
+}
+
+func actionOutputURI(runOutputBase, actionName string) string {
+ if runOutputBase == "" {
+ return ""
+ }
+ return strings.TrimRight(runOutputBase, "/") + "/" + actionName
+}
+
+func getPhaseFromConditions(taskAction *executorv1.TaskAction) common.ActionPhase {
+ for _, cond := range taskAction.Status.Conditions {
+ switch cond.Type {
+ case string(executorv1.ConditionTypeSucceeded):
+ if cond.Status == "True" {
+ return common.ActionPhase_ACTION_PHASE_SUCCEEDED
+ }
+ case string(executorv1.ConditionTypeFailed):
+ if cond.Status == "True" {
+ return common.ActionPhase_ACTION_PHASE_FAILED
+ }
+ case string(executorv1.ConditionTypeProgressing):
+ if cond.Status == "True" {
+ switch cond.Reason {
+ case string(executorv1.ConditionReasonQueued):
+ return common.ActionPhase_ACTION_PHASE_QUEUED
+ case string(executorv1.ConditionReasonInitializing):
+ return common.ActionPhase_ACTION_PHASE_INITIALIZING
+ case string(executorv1.ConditionReasonExecuting):
+ return common.ActionPhase_ACTION_PHASE_RUNNING
+ }
+ }
+ }
+ }
+ return common.ActionPhase_ACTION_PHASE_UNSPECIFIED
+}
diff --git a/actions/service/actions_service_test.go b/actions/service/actions_service_test.go
new file mode 100644
index 00000000000..5a52c487c07
--- /dev/null
+++ b/actions/service/actions_service_test.go
@@ -0,0 +1,172 @@
+package service
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "connectrpc.com/connect"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+
+ "github.com/flyteorg/flyte/v2/actions/service/mocks"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/actions"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/task"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow"
+)
+
+var (
+ testActionID = &common.ActionIdentifier{
+ Run: &common.RunIdentifier{
+ Project: "project",
+ Domain: "domain",
+ Name: "run",
+ },
+ Name: "action",
+ }
+
+ testAction = &actions.Action{
+ ActionId: testActionID,
+ InputUri: "s3://bucket/input",
+ RunOutputBase: "s3://bucket/output",
+ Spec: &actions.Action_Task{
+ Task: &workflow.TaskAction{
+ Spec: &task.TaskSpec{
+ TaskTemplate: &core.TaskTemplate{
+ Type: "container",
+ },
+ },
+ },
+ },
+ }
+)
+
+func TestEnqueue(t *testing.T) {
+ t.Run("success", func(t *testing.T) {
+ m := mocks.NewActionsClientInterface(t)
+ svc := NewActionsService(m)
+
+ m.EXPECT().Enqueue(mock.Anything, testAction, (*task.RunSpec)(nil)).Return(nil)
+
+ resp, err := svc.Enqueue(context.Background(), connect.NewRequest(&actions.EnqueueRequest{
+ Action: testAction,
+ }))
+
+ assert.NoError(t, err)
+ assert.NotNil(t, resp)
+ })
+
+ t.Run("client error returns internal", func(t *testing.T) {
+ m := mocks.NewActionsClientInterface(t)
+ svc := NewActionsService(m)
+
+ m.EXPECT().Enqueue(mock.Anything, testAction, (*task.RunSpec)(nil)).Return(errors.New("k8s error"))
+
+ _, err := svc.Enqueue(context.Background(), connect.NewRequest(&actions.EnqueueRequest{
+ Action: testAction,
+ }))
+
+ assert.Equal(t, connect.CodeInternal, connect.CodeOf(err))
+ })
+}
+
+func TestGetLatestState(t *testing.T) {
+ t.Run("success", func(t *testing.T) {
+ m := mocks.NewActionsClientInterface(t)
+ svc := NewActionsService(m)
+
+ m.EXPECT().GetState(mock.Anything, testActionID).Return(`{"status":"ok"}`, nil)
+
+ resp, err := svc.GetLatestState(context.Background(), connect.NewRequest(&actions.GetLatestStateRequest{
+ ActionId: testActionID,
+ Attempt: 1,
+ }))
+
+ assert.NoError(t, err)
+ assert.Equal(t, `{"status":"ok"}`, resp.Msg.State)
+ })
+
+ t.Run("client error returns not found", func(t *testing.T) {
+ m := mocks.NewActionsClientInterface(t)
+ svc := NewActionsService(m)
+
+ m.EXPECT().GetState(mock.Anything, testActionID).Return("", errors.New("not found"))
+
+ _, err := svc.GetLatestState(context.Background(), connect.NewRequest(&actions.GetLatestStateRequest{
+ ActionId: testActionID,
+ Attempt: 1,
+ }))
+
+ assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err))
+ })
+}
+
+func TestUpdate(t *testing.T) {
+ t.Run("success", func(t *testing.T) {
+ m := mocks.NewActionsClientInterface(t)
+ svc := NewActionsService(m)
+
+ status := &workflow.ActionStatus{Phase: common.ActionPhase_ACTION_PHASE_SUCCEEDED}
+ m.EXPECT().PutState(mock.Anything, testActionID, uint32(1), status, `{}`).Return(nil)
+
+ resp, err := svc.Update(context.Background(), connect.NewRequest(&actions.UpdateRequest{
+ ActionId: testActionID,
+ Attempt: 1,
+ Status: status,
+ State: `{}`,
+ }))
+
+ assert.NoError(t, err)
+ assert.NotNil(t, resp)
+ })
+
+ t.Run("client error returns internal", func(t *testing.T) {
+ m := mocks.NewActionsClientInterface(t)
+ svc := NewActionsService(m)
+
+ status := &workflow.ActionStatus{Phase: common.ActionPhase_ACTION_PHASE_RUNNING}
+ m.EXPECT().PutState(mock.Anything, testActionID, uint32(1), status, `{}`).Return(errors.New("write failed"))
+
+ _, err := svc.Update(context.Background(), connect.NewRequest(&actions.UpdateRequest{
+ ActionId: testActionID,
+ Attempt: 1,
+ Status: status,
+ State: `{}`,
+ }))
+
+ assert.Equal(t, connect.CodeInternal, connect.CodeOf(err))
+ })
+}
+
+func TestAbort(t *testing.T) {
+ t.Run("success", func(t *testing.T) {
+ m := mocks.NewActionsClientInterface(t)
+ svc := NewActionsService(m)
+
+ reason := "user requested"
+ m.EXPECT().AbortAction(mock.Anything, testActionID, &reason).Return(nil)
+
+ resp, err := svc.Abort(context.Background(), connect.NewRequest(&actions.AbortRequest{
+ ActionId: testActionID,
+ Reason: &reason,
+ }))
+
+ assert.NoError(t, err)
+ assert.NotNil(t, resp)
+ })
+
+ t.Run("client error returns internal", func(t *testing.T) {
+ m := mocks.NewActionsClientInterface(t)
+ svc := NewActionsService(m)
+
+ m.EXPECT().AbortAction(mock.Anything, testActionID, (*string)(nil)).Return(errors.New("delete failed"))
+
+ _, err := svc.Abort(context.Background(), connect.NewRequest(&actions.AbortRequest{
+ ActionId: testActionID,
+ }))
+
+ assert.Equal(t, connect.CodeInternal, connect.CodeOf(err))
+ })
+}
diff --git a/actions/service/interfaces.go b/actions/service/interfaces.go
new file mode 100644
index 00000000000..dfd1fcea737
--- /dev/null
+++ b/actions/service/interfaces.go
@@ -0,0 +1,43 @@
+package service
+
+import (
+ "context"
+
+ "github.com/flyteorg/flyte/v2/actions/k8s"
+ executorv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/actions"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/task"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow"
+)
+
+// ActionsClientInterface defines the interface for actions operations.
+// This combines the responsibilities of the queue and state clients.
+type ActionsClientInterface interface {
+ // Enqueue creates a TaskAction CR in Kubernetes.
+ Enqueue(ctx context.Context, action *actions.Action, runSpec *task.RunSpec) error
+
+ // GetState retrieves the state JSON for a TaskAction.
+ GetState(ctx context.Context, actionID *common.ActionIdentifier) (string, error)
+
+ // PutState updates the state and status of a TaskAction.
+ PutState(ctx context.Context, actionID *common.ActionIdentifier, attempt uint32, status *workflow.ActionStatus, stateJSON string) error
+
+ // AbortAction aborts a queued or running action, cascading to descendants.
+ AbortAction(ctx context.Context, actionID *common.ActionIdentifier, reason *string) error
+
+ // ListChildActions lists all TaskActions that are children of the given parent action.
+ ListChildActions(ctx context.Context, parentActionID *common.ActionIdentifier) ([]*executorv1.TaskAction, error)
+
+ // Subscribe creates a new subscription channel for action updates for the given parent action name.
+ Subscribe(parentActionName string) chan *k8s.ActionUpdate
+
+ // Unsubscribe removes the given channel from the subscription list for the parent action name.
+ Unsubscribe(parentActionName string, ch chan *k8s.ActionUpdate)
+
+ // StartWatching starts watching TaskAction resources.
+ StartWatching(ctx context.Context) error
+
+ // StopWatching stops the TaskAction watcher.
+ StopWatching()
+}
diff --git a/actions/service/mocks/mocks.go b/actions/service/mocks/mocks.go
new file mode 100644
index 00000000000..75fa46120ba
--- /dev/null
+++ b/actions/service/mocks/mocks.go
@@ -0,0 +1,562 @@
+// Code generated by mockery; DO NOT EDIT.
+// github.com/vektra/mockery
+// template: testify
+
+package mocks
+
+import (
+ "context"
+
+ "github.com/flyteorg/flyte/v2/actions/k8s"
+ "github.com/flyteorg/flyte/v2/executor/api/v1"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/actions"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/task"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow"
+ mock "github.com/stretchr/testify/mock"
+)
+
+// NewActionsClientInterface creates a new instance of ActionsClientInterface. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+// The first argument is typically a *testing.T value.
+func NewActionsClientInterface(t interface {
+ mock.TestingT
+ Cleanup(func())
+}) *ActionsClientInterface {
+ mock := &ActionsClientInterface{}
+ mock.Mock.Test(t)
+
+ t.Cleanup(func() { mock.AssertExpectations(t) })
+
+ return mock
+}
+
+// ActionsClientInterface is an autogenerated mock type for the ActionsClientInterface type
+type ActionsClientInterface struct {
+ mock.Mock
+}
+
+type ActionsClientInterface_Expecter struct {
+ mock *mock.Mock
+}
+
+func (_m *ActionsClientInterface) EXPECT() *ActionsClientInterface_Expecter {
+ return &ActionsClientInterface_Expecter{mock: &_m.Mock}
+}
+
+// AbortAction provides a mock function for the type ActionsClientInterface
+func (_mock *ActionsClientInterface) AbortAction(ctx context.Context, actionID *common.ActionIdentifier, reason *string) error {
+ ret := _mock.Called(ctx, actionID, reason)
+
+ if len(ret) == 0 {
+ panic("no return value specified for AbortAction")
+ }
+
+ var r0 error
+ if returnFunc, ok := ret.Get(0).(func(context.Context, *common.ActionIdentifier, *string) error); ok {
+ r0 = returnFunc(ctx, actionID, reason)
+ } else {
+ r0 = ret.Error(0)
+ }
+ return r0
+}
+
+// ActionsClientInterface_AbortAction_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AbortAction'
+type ActionsClientInterface_AbortAction_Call struct {
+ *mock.Call
+}
+
+// AbortAction is a helper method to define mock.On call
+// - ctx context.Context
+// - actionID *common.ActionIdentifier
+// - reason *string
+func (_e *ActionsClientInterface_Expecter) AbortAction(ctx interface{}, actionID interface{}, reason interface{}) *ActionsClientInterface_AbortAction_Call {
+ return &ActionsClientInterface_AbortAction_Call{Call: _e.mock.On("AbortAction", ctx, actionID, reason)}
+}
+
+func (_c *ActionsClientInterface_AbortAction_Call) Run(run func(ctx context.Context, actionID *common.ActionIdentifier, reason *string)) *ActionsClientInterface_AbortAction_Call {
+ _c.Call.Run(func(args mock.Arguments) {
+ var arg0 context.Context
+ if args[0] != nil {
+ arg0 = args[0].(context.Context)
+ }
+ var arg1 *common.ActionIdentifier
+ if args[1] != nil {
+ arg1 = args[1].(*common.ActionIdentifier)
+ }
+ var arg2 *string
+ if args[2] != nil {
+ arg2 = args[2].(*string)
+ }
+ run(
+ arg0,
+ arg1,
+ arg2,
+ )
+ })
+ return _c
+}
+
+func (_c *ActionsClientInterface_AbortAction_Call) Return(err error) *ActionsClientInterface_AbortAction_Call {
+ _c.Call.Return(err)
+ return _c
+}
+
+func (_c *ActionsClientInterface_AbortAction_Call) RunAndReturn(run func(ctx context.Context, actionID *common.ActionIdentifier, reason *string) error) *ActionsClientInterface_AbortAction_Call {
+ _c.Call.Return(run)
+ return _c
+}
+
+// Enqueue provides a mock function for the type ActionsClientInterface
+func (_mock *ActionsClientInterface) Enqueue(ctx context.Context, action *actions.Action, runSpec *task.RunSpec) error {
+ ret := _mock.Called(ctx, action, runSpec)
+
+ if len(ret) == 0 {
+ panic("no return value specified for Enqueue")
+ }
+
+ var r0 error
+ if returnFunc, ok := ret.Get(0).(func(context.Context, *actions.Action, *task.RunSpec) error); ok {
+ r0 = returnFunc(ctx, action, runSpec)
+ } else {
+ r0 = ret.Error(0)
+ }
+ return r0
+}
+
+// ActionsClientInterface_Enqueue_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Enqueue'
+type ActionsClientInterface_Enqueue_Call struct {
+ *mock.Call
+}
+
+// Enqueue is a helper method to define mock.On call
+// - ctx context.Context
+// - action *actions.Action
+// - runSpec *task.RunSpec
+func (_e *ActionsClientInterface_Expecter) Enqueue(ctx interface{}, action interface{}, runSpec interface{}) *ActionsClientInterface_Enqueue_Call {
+ return &ActionsClientInterface_Enqueue_Call{Call: _e.mock.On("Enqueue", ctx, action, runSpec)}
+}
+
+func (_c *ActionsClientInterface_Enqueue_Call) Run(run func(ctx context.Context, action *actions.Action, runSpec *task.RunSpec)) *ActionsClientInterface_Enqueue_Call {
+ _c.Call.Run(func(args mock.Arguments) {
+ var arg0 context.Context
+ if args[0] != nil {
+ arg0 = args[0].(context.Context)
+ }
+ var arg1 *actions.Action
+ if args[1] != nil {
+ arg1 = args[1].(*actions.Action)
+ }
+ var arg2 *task.RunSpec
+ if args[2] != nil {
+ arg2 = args[2].(*task.RunSpec)
+ }
+ run(
+ arg0,
+ arg1,
+ arg2,
+ )
+ })
+ return _c
+}
+
+func (_c *ActionsClientInterface_Enqueue_Call) Return(err error) *ActionsClientInterface_Enqueue_Call {
+ _c.Call.Return(err)
+ return _c
+}
+
+func (_c *ActionsClientInterface_Enqueue_Call) RunAndReturn(run func(ctx context.Context, action *actions.Action, runSpec *task.RunSpec) error) *ActionsClientInterface_Enqueue_Call {
+ _c.Call.Return(run)
+ return _c
+}
+
+// GetState provides a mock function for the type ActionsClientInterface
+func (_mock *ActionsClientInterface) GetState(ctx context.Context, actionID *common.ActionIdentifier) (string, error) {
+ ret := _mock.Called(ctx, actionID)
+
+ if len(ret) == 0 {
+ panic("no return value specified for GetState")
+ }
+
+ var r0 string
+ var r1 error
+ if returnFunc, ok := ret.Get(0).(func(context.Context, *common.ActionIdentifier) (string, error)); ok {
+ return returnFunc(ctx, actionID)
+ }
+ if returnFunc, ok := ret.Get(0).(func(context.Context, *common.ActionIdentifier) string); ok {
+ r0 = returnFunc(ctx, actionID)
+ } else {
+ r0 = ret.Get(0).(string)
+ }
+ if returnFunc, ok := ret.Get(1).(func(context.Context, *common.ActionIdentifier) error); ok {
+ r1 = returnFunc(ctx, actionID)
+ } else {
+ r1 = ret.Error(1)
+ }
+ return r0, r1
+}
+
+// ActionsClientInterface_GetState_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetState'
+type ActionsClientInterface_GetState_Call struct {
+ *mock.Call
+}
+
+// GetState is a helper method to define mock.On call
+// - ctx context.Context
+// - actionID *common.ActionIdentifier
+func (_e *ActionsClientInterface_Expecter) GetState(ctx interface{}, actionID interface{}) *ActionsClientInterface_GetState_Call {
+ return &ActionsClientInterface_GetState_Call{Call: _e.mock.On("GetState", ctx, actionID)}
+}
+
+func (_c *ActionsClientInterface_GetState_Call) Run(run func(ctx context.Context, actionID *common.ActionIdentifier)) *ActionsClientInterface_GetState_Call {
+ _c.Call.Run(func(args mock.Arguments) {
+ var arg0 context.Context
+ if args[0] != nil {
+ arg0 = args[0].(context.Context)
+ }
+ var arg1 *common.ActionIdentifier
+ if args[1] != nil {
+ arg1 = args[1].(*common.ActionIdentifier)
+ }
+ run(
+ arg0,
+ arg1,
+ )
+ })
+ return _c
+}
+
+func (_c *ActionsClientInterface_GetState_Call) Return(s string, err error) *ActionsClientInterface_GetState_Call {
+ _c.Call.Return(s, err)
+ return _c
+}
+
+func (_c *ActionsClientInterface_GetState_Call) RunAndReturn(run func(ctx context.Context, actionID *common.ActionIdentifier) (string, error)) *ActionsClientInterface_GetState_Call {
+ _c.Call.Return(run)
+ return _c
+}
+
+// ListChildActions provides a mock function for the type ActionsClientInterface
+func (_mock *ActionsClientInterface) ListChildActions(ctx context.Context, parentActionID *common.ActionIdentifier) ([]*v1.TaskAction, error) {
+ ret := _mock.Called(ctx, parentActionID)
+
+ if len(ret) == 0 {
+ panic("no return value specified for ListChildActions")
+ }
+
+ var r0 []*v1.TaskAction
+ var r1 error
+ if returnFunc, ok := ret.Get(0).(func(context.Context, *common.ActionIdentifier) ([]*v1.TaskAction, error)); ok {
+ return returnFunc(ctx, parentActionID)
+ }
+ if returnFunc, ok := ret.Get(0).(func(context.Context, *common.ActionIdentifier) []*v1.TaskAction); ok {
+ r0 = returnFunc(ctx, parentActionID)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).([]*v1.TaskAction)
+ }
+ }
+ if returnFunc, ok := ret.Get(1).(func(context.Context, *common.ActionIdentifier) error); ok {
+ r1 = returnFunc(ctx, parentActionID)
+ } else {
+ r1 = ret.Error(1)
+ }
+ return r0, r1
+}
+
+// ActionsClientInterface_ListChildActions_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListChildActions'
+type ActionsClientInterface_ListChildActions_Call struct {
+ *mock.Call
+}
+
+// ListChildActions is a helper method to define mock.On call
+// - ctx context.Context
+// - parentActionID *common.ActionIdentifier
+func (_e *ActionsClientInterface_Expecter) ListChildActions(ctx interface{}, parentActionID interface{}) *ActionsClientInterface_ListChildActions_Call {
+ return &ActionsClientInterface_ListChildActions_Call{Call: _e.mock.On("ListChildActions", ctx, parentActionID)}
+}
+
+func (_c *ActionsClientInterface_ListChildActions_Call) Run(run func(ctx context.Context, parentActionID *common.ActionIdentifier)) *ActionsClientInterface_ListChildActions_Call {
+ _c.Call.Run(func(args mock.Arguments) {
+ var arg0 context.Context
+ if args[0] != nil {
+ arg0 = args[0].(context.Context)
+ }
+ var arg1 *common.ActionIdentifier
+ if args[1] != nil {
+ arg1 = args[1].(*common.ActionIdentifier)
+ }
+ run(
+ arg0,
+ arg1,
+ )
+ })
+ return _c
+}
+
+func (_c *ActionsClientInterface_ListChildActions_Call) Return(taskActions []*v1.TaskAction, err error) *ActionsClientInterface_ListChildActions_Call {
+ _c.Call.Return(taskActions, err)
+ return _c
+}
+
+func (_c *ActionsClientInterface_ListChildActions_Call) RunAndReturn(run func(ctx context.Context, parentActionID *common.ActionIdentifier) ([]*v1.TaskAction, error)) *ActionsClientInterface_ListChildActions_Call {
+ _c.Call.Return(run)
+ return _c
+}
+
+// PutState provides a mock function for the type ActionsClientInterface
+func (_mock *ActionsClientInterface) PutState(ctx context.Context, actionID *common.ActionIdentifier, attempt uint32, status *workflow.ActionStatus, stateJSON string) error {
+ ret := _mock.Called(ctx, actionID, attempt, status, stateJSON)
+
+ if len(ret) == 0 {
+ panic("no return value specified for PutState")
+ }
+
+ var r0 error
+ if returnFunc, ok := ret.Get(0).(func(context.Context, *common.ActionIdentifier, uint32, *workflow.ActionStatus, string) error); ok {
+ r0 = returnFunc(ctx, actionID, attempt, status, stateJSON)
+ } else {
+ r0 = ret.Error(0)
+ }
+ return r0
+}
+
+// ActionsClientInterface_PutState_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PutState'
+type ActionsClientInterface_PutState_Call struct {
+ *mock.Call
+}
+
+// PutState is a helper method to define mock.On call
+// - ctx context.Context
+// - actionID *common.ActionIdentifier
+// - attempt uint32
+// - status *workflow.ActionStatus
+// - stateJSON string
+func (_e *ActionsClientInterface_Expecter) PutState(ctx interface{}, actionID interface{}, attempt interface{}, status interface{}, stateJSON interface{}) *ActionsClientInterface_PutState_Call {
+ return &ActionsClientInterface_PutState_Call{Call: _e.mock.On("PutState", ctx, actionID, attempt, status, stateJSON)}
+}
+
+func (_c *ActionsClientInterface_PutState_Call) Run(run func(ctx context.Context, actionID *common.ActionIdentifier, attempt uint32, status *workflow.ActionStatus, stateJSON string)) *ActionsClientInterface_PutState_Call {
+ _c.Call.Run(func(args mock.Arguments) {
+ var arg0 context.Context
+ if args[0] != nil {
+ arg0 = args[0].(context.Context)
+ }
+ var arg1 *common.ActionIdentifier
+ if args[1] != nil {
+ arg1 = args[1].(*common.ActionIdentifier)
+ }
+ var arg2 uint32
+ if args[2] != nil {
+ arg2 = args[2].(uint32)
+ }
+ var arg3 *workflow.ActionStatus
+ if args[3] != nil {
+ arg3 = args[3].(*workflow.ActionStatus)
+ }
+ var arg4 string
+ if args[4] != nil {
+ arg4 = args[4].(string)
+ }
+ run(
+ arg0,
+ arg1,
+ arg2,
+ arg3,
+ arg4,
+ )
+ })
+ return _c
+}
+
+func (_c *ActionsClientInterface_PutState_Call) Return(err error) *ActionsClientInterface_PutState_Call {
+ _c.Call.Return(err)
+ return _c
+}
+
+func (_c *ActionsClientInterface_PutState_Call) RunAndReturn(run func(ctx context.Context, actionID *common.ActionIdentifier, attempt uint32, status *workflow.ActionStatus, stateJSON string) error) *ActionsClientInterface_PutState_Call {
+ _c.Call.Return(run)
+ return _c
+}
+
+// StartWatching provides a mock function for the type ActionsClientInterface
+func (_mock *ActionsClientInterface) StartWatching(ctx context.Context) error {
+ ret := _mock.Called(ctx)
+
+ if len(ret) == 0 {
+ panic("no return value specified for StartWatching")
+ }
+
+ var r0 error
+ if returnFunc, ok := ret.Get(0).(func(context.Context) error); ok {
+ r0 = returnFunc(ctx)
+ } else {
+ r0 = ret.Error(0)
+ }
+ return r0
+}
+
+// ActionsClientInterface_StartWatching_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'StartWatching'
+type ActionsClientInterface_StartWatching_Call struct {
+ *mock.Call
+}
+
+// StartWatching is a helper method to define mock.On call
+// - ctx context.Context
+func (_e *ActionsClientInterface_Expecter) StartWatching(ctx interface{}) *ActionsClientInterface_StartWatching_Call {
+ return &ActionsClientInterface_StartWatching_Call{Call: _e.mock.On("StartWatching", ctx)}
+}
+
+func (_c *ActionsClientInterface_StartWatching_Call) Run(run func(ctx context.Context)) *ActionsClientInterface_StartWatching_Call {
+ _c.Call.Run(func(args mock.Arguments) {
+ var arg0 context.Context
+ if args[0] != nil {
+ arg0 = args[0].(context.Context)
+ }
+ run(
+ arg0,
+ )
+ })
+ return _c
+}
+
+func (_c *ActionsClientInterface_StartWatching_Call) Return(err error) *ActionsClientInterface_StartWatching_Call {
+ _c.Call.Return(err)
+ return _c
+}
+
+func (_c *ActionsClientInterface_StartWatching_Call) RunAndReturn(run func(ctx context.Context) error) *ActionsClientInterface_StartWatching_Call {
+ _c.Call.Return(run)
+ return _c
+}
+
+// StopWatching provides a mock function for the type ActionsClientInterface
+func (_mock *ActionsClientInterface) StopWatching() {
+ _mock.Called()
+ return
+}
+
+// ActionsClientInterface_StopWatching_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'StopWatching'
+type ActionsClientInterface_StopWatching_Call struct {
+ *mock.Call
+}
+
+// StopWatching is a helper method to define mock.On call
+func (_e *ActionsClientInterface_Expecter) StopWatching() *ActionsClientInterface_StopWatching_Call {
+ return &ActionsClientInterface_StopWatching_Call{Call: _e.mock.On("StopWatching")}
+}
+
+func (_c *ActionsClientInterface_StopWatching_Call) Run(run func()) *ActionsClientInterface_StopWatching_Call {
+ _c.Call.Run(func(args mock.Arguments) {
+ run()
+ })
+ return _c
+}
+
+func (_c *ActionsClientInterface_StopWatching_Call) Return() *ActionsClientInterface_StopWatching_Call {
+ _c.Call.Return()
+ return _c
+}
+
+func (_c *ActionsClientInterface_StopWatching_Call) RunAndReturn(run func()) *ActionsClientInterface_StopWatching_Call {
+ _c.Run(run)
+ return _c
+}
+
+// Subscribe provides a mock function for the type ActionsClientInterface
+func (_mock *ActionsClientInterface) Subscribe(parentActionName string) chan *k8s.ActionUpdate {
+ ret := _mock.Called(parentActionName)
+
+ if len(ret) == 0 {
+ panic("no return value specified for Subscribe")
+ }
+
+ var r0 chan *k8s.ActionUpdate
+ if returnFunc, ok := ret.Get(0).(func(string) chan *k8s.ActionUpdate); ok {
+ r0 = returnFunc(parentActionName)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).(chan *k8s.ActionUpdate)
+ }
+ }
+ return r0
+}
+
+// ActionsClientInterface_Subscribe_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Subscribe'
+type ActionsClientInterface_Subscribe_Call struct {
+ *mock.Call
+}
+
+// Subscribe is a helper method to define mock.On call
+// - parentActionName string
+func (_e *ActionsClientInterface_Expecter) Subscribe(parentActionName interface{}) *ActionsClientInterface_Subscribe_Call {
+ return &ActionsClientInterface_Subscribe_Call{Call: _e.mock.On("Subscribe", parentActionName)}
+}
+
+func (_c *ActionsClientInterface_Subscribe_Call) Run(run func(parentActionName string)) *ActionsClientInterface_Subscribe_Call {
+ _c.Call.Run(func(args mock.Arguments) {
+ var arg0 string
+ if args[0] != nil {
+ arg0 = args[0].(string)
+ }
+ run(
+ arg0,
+ )
+ })
+ return _c
+}
+
+func (_c *ActionsClientInterface_Subscribe_Call) Return(actionUpdateCh chan *k8s.ActionUpdate) *ActionsClientInterface_Subscribe_Call {
+ _c.Call.Return(actionUpdateCh)
+ return _c
+}
+
+func (_c *ActionsClientInterface_Subscribe_Call) RunAndReturn(run func(parentActionName string) chan *k8s.ActionUpdate) *ActionsClientInterface_Subscribe_Call {
+ _c.Call.Return(run)
+ return _c
+}
+
+// Unsubscribe provides a mock function for the type ActionsClientInterface
+func (_mock *ActionsClientInterface) Unsubscribe(parentActionName string, ch chan *k8s.ActionUpdate) {
+ _mock.Called(parentActionName, ch)
+ return
+}
+
+// ActionsClientInterface_Unsubscribe_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Unsubscribe'
+type ActionsClientInterface_Unsubscribe_Call struct {
+ *mock.Call
+}
+
+// Unsubscribe is a helper method to define mock.On call
+// - parentActionName string
+// - ch chan *k8s.ActionUpdate
+func (_e *ActionsClientInterface_Expecter) Unsubscribe(parentActionName interface{}, ch interface{}) *ActionsClientInterface_Unsubscribe_Call {
+ return &ActionsClientInterface_Unsubscribe_Call{Call: _e.mock.On("Unsubscribe", parentActionName, ch)}
+}
+
+func (_c *ActionsClientInterface_Unsubscribe_Call) Run(run func(parentActionName string, ch chan *k8s.ActionUpdate)) *ActionsClientInterface_Unsubscribe_Call {
+ _c.Call.Run(func(args mock.Arguments) {
+ var arg0 string
+ if args[0] != nil {
+ arg0 = args[0].(string)
+ }
+ var arg1 chan *k8s.ActionUpdate
+ if args[1] != nil {
+ arg1 = args[1].(chan *k8s.ActionUpdate)
+ }
+ run(
+ arg0,
+ arg1,
+ )
+ })
+ return _c
+}
+
+func (_c *ActionsClientInterface_Unsubscribe_Call) Return() *ActionsClientInterface_Unsubscribe_Call {
+ _c.Call.Return()
+ return _c
+}
+
+func (_c *ActionsClientInterface_Unsubscribe_Call) RunAndReturn(run func(parentActionName string, ch chan *k8s.ActionUpdate)) *ActionsClientInterface_Unsubscribe_Call {
+ _c.Run(run)
+ return _c
+}
diff --git a/actions/setup.go b/actions/setup.go
new file mode 100644
index 00000000000..6f20d863403
--- /dev/null
+++ b/actions/setup.go
@@ -0,0 +1,60 @@
+package actions
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+
+ "github.com/flyteorg/flyte/v2/actions/config"
+ actionsk8s "github.com/flyteorg/flyte/v2/actions/k8s"
+ "github.com/flyteorg/flyte/v2/actions/service"
+ "github.com/flyteorg/flyte/v2/flytestdlib/app"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/actions/actionsconnect"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow/workflowconnect"
+)
+
+// Setup registers the ActionsService handler on the SetupContext mux.
+// Requires sc.K8sClient and sc.Namespace to be set.
+func Setup(ctx context.Context, sc *app.SetupContext) error {
+ cfg := config.GetConfig()
+
+ if err := actionsk8s.InitScheme(); err != nil {
+ return fmt.Errorf("actions: failed to initialize scheme: %w", err)
+ }
+
+ runServiceURL := cfg.RunServiceURL
+ if sc.BaseURL != "" {
+ runServiceURL = sc.BaseURL
+ }
+ runClient := workflowconnect.NewInternalRunServiceClient(http.DefaultClient, runServiceURL)
+
+ actionsClient := actionsk8s.NewActionsClient(
+ sc.K8sClient,
+ sc.K8sCache,
+ sc.Namespace,
+ cfg.WatchBufferSize,
+ cfg.WatchWorkers,
+ runClient,
+ cfg.RecordFilterSize,
+ sc.Scope,
+ )
+ logger.Infof(ctx, "Actions K8s client initialized for namespace: %s", sc.Namespace)
+
+ if err := actionsClient.StartWatching(ctx); err != nil {
+ return fmt.Errorf("actions: failed to start TaskAction watcher: %w", err)
+ }
+ sc.AddWorker("actions-watcher", func(ctx context.Context) error {
+ <-ctx.Done()
+ actionsClient.StopWatching()
+ return nil
+ })
+
+ actionsSvc := service.NewActionsService(actionsClient)
+
+ path, handler := actionsconnect.NewActionsServiceHandler(actionsSvc)
+ sc.Mux.Handle(path, handler)
+ logger.Infof(ctx, "Mounted ActionsService at %s", path)
+
+ return nil
+}
diff --git a/app/config/config.go b/app/config/config.go
new file mode 100644
index 00000000000..6532f30b5c0
--- /dev/null
+++ b/app/config/config.go
@@ -0,0 +1,22 @@
+package config
+
+import "time"
+
+// AppConfig holds configuration for the control plane AppService.
+type AppConfig struct {
+ // InternalAppServiceURL is the base URL of the InternalAppService (data plane).
+ // In unified mode this is overridden by the shared mux BaseURL.
+ InternalAppServiceURL string `json:"internalAppServiceUrl" pflag:",URL of the internal app service"`
+
+ // CacheTTL is the TTL for the in-memory app status cache.
+ // Defaults to 30s. Set to 0 to disable caching.
+ CacheTTL time.Duration `json:"cacheTtl" pflag:",TTL for app status cache"`
+}
+
+// DefaultAppConfig returns the default control plane AppConfig.
+func DefaultAppConfig() *AppConfig {
+ return &AppConfig{
+ InternalAppServiceURL: "http://localhost:8091",
+ CacheTTL: 30 * time.Second,
+ }
+}
diff --git a/app/internal/config/config.go b/app/internal/config/config.go
new file mode 100644
index 00000000000..17e9e01dc23
--- /dev/null
+++ b/app/internal/config/config.go
@@ -0,0 +1,19 @@
+package config
+
+import "time"
+
+// InternalAppConfig holds configuration for the data plane app deployment controller.
+type InternalAppConfig struct {
+ // Enabled controls whether the app deployment controller is started.
+ Enabled bool `json:"enabled" pflag:",Enable app deployment controller"`
+
+ // BaseDomain is the base domain used to generate public URLs for apps.
+ // Apps are exposed at "{name}-{project}-{domain}.{base_domain}".
+ BaseDomain string `json:"baseDomain" pflag:",Base domain for app public URLs"`
+
+ // DefaultRequestTimeout is the request timeout applied to apps that don't specify one.
+ DefaultRequestTimeout time.Duration `json:"defaultRequestTimeout" pflag:",Default request timeout for apps"`
+
+ // MaxRequestTimeout is the hard cap on request timeout (Knative max is 3600s).
+ MaxRequestTimeout time.Duration `json:"maxRequestTimeout" pflag:",Maximum allowed request timeout for apps"`
+}
diff --git a/app/internal/k8s/app_client.go b/app/internal/k8s/app_client.go
new file mode 100644
index 00000000000..474823529e6
--- /dev/null
+++ b/app/internal/k8s/app_client.go
@@ -0,0 +1,784 @@
+package k8s
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "math"
+ "strings"
+ "time"
+
+ "google.golang.org/protobuf/proto"
+ timestamppb "google.golang.org/protobuf/types/known/timestamppb"
+ corev1 "k8s.io/api/core/v1"
+ k8serrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+ k8swatch "k8s.io/apimachinery/pkg/watch"
+ servingv1 "knative.dev/serving/pkg/apis/serving/v1"
+ ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+
+ "github.com/flyteorg/flyte/v2/app/internal/config"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ flyteapp "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app"
+)
+
+const (
+ labelAppManaged = "flyte.org/app-managed"
+ labelProject = "flyte.org/project"
+ labelDomain = "flyte.org/domain"
+ labelAppName = "flyte.org/app-name"
+
+ annotationSpecSHA = "flyte.org/spec-sha"
+ annotationAppID = "flyte.org/app-id"
+
+ maxScaleZero = "0"
+
+ // maxKServiceNameLen is the Kubernetes DNS label limit.
+ maxKServiceNameLen = 63
+)
+
+// AppK8sClientInterface defines the KService lifecycle operations for the App service.
+type AppK8sClientInterface interface {
+ // Deploy creates or updates the KService for the given app. Idempotent — skips
+ // the update if the spec SHA annotation is unchanged.
+ Deploy(ctx context.Context, app *flyteapp.App) error
+
+ // Stop scales the KService to zero by setting max-scale=0. The KService CRD
+ // is kept so the app can be restarted later.
+ Stop(ctx context.Context, appID *flyteapp.Identifier) error
+
+ // GetStatus reads the KService and maps its conditions to a DeploymentStatus.
+ // Returns a not-found error (checkable with k8serrors.IsNotFound) if the KService does not exist.
+ GetStatus(ctx context.Context, appID *flyteapp.Identifier) (*flyteapp.Status, error)
+
+ // List returns apps for the given project/domain scope with optional pagination.
+ // If appName is non-empty, only the app with that name is returned.
+ // limit=0 means no limit. token is the K8s continue token from a previous call.
+ // Returns the apps, the continue token for the next page (empty if last page), and any error.
+ List(ctx context.Context, project, domain, appName string, limit uint32, token string) ([]*flyteapp.App, string, error)
+
+ // Delete removes the KService CRD entirely. The app must be re-created from scratch.
+ // Use Stop to scale to zero while preserving the KService.
+ Delete(ctx context.Context, appID *flyteapp.Identifier) error
+
+ // GetReplicas lists the pods (replicas) currently backing the given app.
+ GetReplicas(ctx context.Context, appID *flyteapp.Identifier) ([]*flyteapp.Replica, error)
+
+ // DeleteReplica force-deletes a specific pod. Knative will replace it automatically.
+ DeleteReplica(ctx context.Context, replicaID *flyteapp.ReplicaIdentifier) error
+
+ // Watch returns a channel of WatchResponse events for KServices matching the
+ // given project/domain scope. If appName is non-empty, only events for that
+ // specific app are returned. The channel is closed when ctx is cancelled.
+ Watch(ctx context.Context, project, domain, appName string) (<-chan *flyteapp.WatchResponse, error)
+}
+
+// AppK8sClient implements AppK8sClientInterface using controller-runtime.
+type AppK8sClient struct {
+ k8sClient client.WithWatch
+ cache ctrlcache.Cache
+ cfg *config.InternalAppConfig
+}
+
+// NewAppK8sClient creates a new AppK8sClient.
+func NewAppK8sClient(k8sClient client.WithWatch, cache ctrlcache.Cache, cfg *config.InternalAppConfig) *AppK8sClient {
+ return &AppK8sClient{
+ k8sClient: k8sClient,
+ cache: cache,
+ cfg: cfg,
+ }
+}
+
+// appNamespace returns the K8s namespace for a given project/domain pair.
+// Follows the same convention as the Actions and Secret services: "{project}-{domain}".
+func appNamespace(project, domain string) string {
+ return fmt.Sprintf("%s-%s", project, domain)
+}
+
+// Deploy creates or updates the KService for the given app.
+func (c *AppK8sClient) Deploy(ctx context.Context, app *flyteapp.App) error {
+ appID := app.GetMetadata().GetId()
+ ns := appNamespace(appID.GetProject(), appID.GetDomain())
+ name := kserviceName(appID)
+
+ ksvc, err := c.buildKService(app)
+ if err != nil {
+ return fmt.Errorf("failed to build KService for app %s: %w", name, err)
+ }
+
+ existing := &servingv1.Service{}
+ err = c.k8sClient.Get(ctx, client.ObjectKey{Name: name, Namespace: ns}, existing)
+ if k8serrors.IsNotFound(err) {
+ if err := c.k8sClient.Create(ctx, ksvc); err != nil {
+ return fmt.Errorf("failed to create KService %s: %w", name, err)
+ }
+ logger.Infof(ctx, "Created KService %s/%s", ns, name)
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("failed to get KService %s: %w", name, err)
+ }
+
+ // Skip update if spec has not changed.
+ if existing.Annotations[annotationSpecSHA] == ksvc.Annotations[annotationSpecSHA] {
+ logger.Debugf(ctx, "KService %s/%s spec unchanged, skipping update", ns, name)
+ return nil
+ }
+
+ existing.Spec = ksvc.Spec
+ existing.Labels = ksvc.Labels
+ existing.Annotations = ksvc.Annotations
+ if err := c.k8sClient.Update(ctx, existing); err != nil {
+ return fmt.Errorf("failed to update KService %s: %w", name, err)
+ }
+ logger.Infof(ctx, "Updated KService %s/%s", ns, name)
+ return nil
+}
+
+// Stop sets max-scale=0 on the KService, scaling it to zero without deleting it.
+func (c *AppK8sClient) Stop(ctx context.Context, appID *flyteapp.Identifier) error {
+ ns := appNamespace(appID.GetProject(), appID.GetDomain())
+ name := kserviceName(appID)
+ patch := []byte(`{"spec":{"template":{"metadata":{"annotations":{"autoscaling.knative.dev/max-scale":"0"}}}}}`)
+ ksvc := &servingv1.Service{}
+ ksvc.Name = name
+ ksvc.Namespace = ns
+ if err := c.k8sClient.Patch(ctx, ksvc, client.RawPatch(types.MergePatchType, patch)); err != nil {
+ if k8serrors.IsNotFound(err) {
+ // Already stopped/deleted — treat as success.
+ return nil
+ }
+ return fmt.Errorf("failed to patch KService %s to stop: %w", name, err)
+ }
+ logger.Infof(ctx, "Stopped KService %s/%s (max-scale=0)", ns, name)
+ return nil
+}
+
+// Delete removes the KService CRD for the given app entirely.
+func (c *AppK8sClient) Delete(ctx context.Context, appID *flyteapp.Identifier) error {
+ ns := appNamespace(appID.GetProject(), appID.GetDomain())
+ name := kserviceName(appID)
+ ksvc := &servingv1.Service{}
+ ksvc.Name = name
+ ksvc.Namespace = ns
+ if err := c.k8sClient.Delete(ctx, ksvc); err != nil {
+ if k8serrors.IsNotFound(err) {
+ return nil
+ }
+ return fmt.Errorf("failed to delete KService %s: %w", name, err)
+ }
+ logger.Infof(ctx, "Deleted KService %s/%s", ns, name)
+ return nil
+}
+
+// watchBackoff controls the reconnect timing for Watch. Declared as vars so
+// tests can override them without sleeping.
+var (
+ watchBackoffInitial = 1 * time.Second
+ watchBackoffMax = 30 * time.Second
+ watchBackoffFactor = 2.0
+)
+
+// watchState holds mutable reconnect state for a single Watch call.
+// It is goroutine-local — no mutex needed.
+type watchState struct {
+ // lastResourceVersion is the most recent RV seen from any event or Bookmark.
+ // Passed to openWatch on reconnect so K8s resumes from exactly where we left off.
+ lastResourceVersion string
+ backoff time.Duration
+ consecutiveErrors int
+}
+
+func (s *watchState) nextBackoff() time.Duration {
+ d := s.backoff
+ if d == 0 {
+ d = watchBackoffInitial
+ }
+ s.backoff = time.Duration(math.Min(float64(d)*watchBackoffFactor, float64(watchBackoffMax)))
+ return d
+}
+
+func (s *watchState) resetBackoff() {
+ s.backoff = watchBackoffInitial
+ s.consecutiveErrors = 0
+}
+
+// Watch returns a channel of WatchResponse events for KServices in the given
+// project/domain scope. If appName is non-empty, only events for that specific
+// app are returned. The channel is closed only when ctx is cancelled.
+//
+// The goroutine reconnects transparently when the underlying K8s watch closes
+// unexpectedly, tracking resourceVersion to resume without gaps or replays.
+func (c *AppK8sClient) Watch(ctx context.Context, project, domain, appName string) (<-chan *flyteapp.WatchResponse, error) {
+ ns := appNamespace(project, domain)
+ labels := map[string]string{labelAppManaged: "true"}
+ if appName != "" {
+ labels[labelAppName] = strings.ToLower(appName)
+ }
+
+ // Open the first watcher eagerly so initial errors (RBAC, missing CRD) are
+ // returned synchronously before spawning the goroutine.
+ watcher, err := c.openWatch(ctx, ns, labels, "")
+ if err != nil {
+ return nil, err
+ }
+
+ ch := make(chan *flyteapp.WatchResponse, 64)
+ go c.watchLoop(ctx, ns, labels, watcher, ch)
+ return ch, nil
+}
+
+// openWatch starts a K8s watch from resourceVersion (empty = watch from now).
+// AllowWatchBookmarks is always set so K8s sends Bookmark events on every session,
+// keeping lastResourceVersion current even when no objects change.
+func (c *AppK8sClient) openWatch(ctx context.Context, ns string, labels map[string]string, resourceVersion string) (k8swatch.Interface, error) {
+ rawOpts := &metav1.ListOptions{AllowWatchBookmarks: true}
+ if resourceVersion != "" {
+ rawOpts.ResourceVersion = resourceVersion
+ }
+ opts := []client.ListOption{
+ client.InNamespace(ns),
+ client.MatchingLabels(labels),
+ &client.ListOptions{Raw: rawOpts},
+ }
+ watcher, err := c.k8sClient.Watch(ctx, &servingv1.ServiceList{}, opts...)
+ if err != nil {
+ return nil, fmt.Errorf("failed to start KService watch in namespace %s: %w", ns, err)
+ }
+ return watcher, nil
+}
+
+// watchLoop is the reconnect loop. It drains watcher until it closes, then
+// reopens. Exponential backoff is applied only on K8s Error events; normal
+// watch timeouts (clean channel close) reconnect immediately. Closes ch only
+// when ctx is cancelled.
+func (c *AppK8sClient) watchLoop(
+ ctx context.Context,
+ ns string,
+ labels map[string]string,
+ watcher k8swatch.Interface,
+ ch chan<- *flyteapp.WatchResponse,
+) {
+ defer close(ch)
+ defer watcher.Stop()
+
+ state := &watchState{backoff: watchBackoffInitial}
+
+ for {
+ reconnect, isError := c.drainWatcher(ctx, watcher, ch, state)
+ if !reconnect {
+ return // ctx cancelled
+ }
+
+ watcher.Stop()
+
+ if isError {
+ state.consecutiveErrors++
+ delay := state.nextBackoff()
+ logger.Warnf(ctx, "KService watch in namespace %s closed with error (attempt %d); reconnecting in %v",
+ ns, state.consecutiveErrors, delay)
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(delay):
+ }
+ } else {
+ // Normal K8s watch timeout — reconnect immediately, no backoff.
+ state.resetBackoff()
+ logger.Debugf(ctx, "KService watch in namespace %s timed out naturally; reconnecting", ns)
+ }
+
+ newWatcher, err := c.openWatch(ctx, ns, labels, state.lastResourceVersion)
+ if err != nil {
+ logger.Errorf(ctx, "Failed to reopen KService watch in namespace %s: %v", ns, err)
+ // Use an immediately-closed watcher so the loop retries with further backoff.
+ watcher = k8swatch.NewEmptyWatch()
+ continue
+ }
+ watcher = newWatcher
+ }
+}
+
+// drainWatcher processes events from watcher until the channel closes or ctx is done.
+// Returns (reconnect, isError): reconnect=false means ctx was cancelled (stop the loop);
+// isError=true means a K8s Error event triggered the close and backoff should be applied.
+func (c *AppK8sClient) drainWatcher(
+ ctx context.Context,
+ watcher k8swatch.Interface,
+ ch chan<- *flyteapp.WatchResponse,
+ state *watchState,
+) (bool, bool) {
+ for {
+ select {
+ case <-ctx.Done():
+ return false, false
+ case event, ok := <-watcher.ResultChan():
+ if !ok {
+ return true, false // normal K8s watch timeout
+ }
+
+ switch event.Type {
+ case k8swatch.Error:
+ if status, ok := event.Object.(*metav1.Status); ok {
+ logger.Warnf(ctx, "KService watch received error event (code=%d reason=%s): %s; will reconnect",
+ status.Code, status.Reason, status.Message)
+ } else {
+ logger.Warnf(ctx, "KService watch received error event (type %T); will reconnect", event.Object)
+ }
+ return true, true // error — apply backoff on reconnect
+ case k8swatch.Bookmark:
+ // Update RV immediately — there is no delivery to confirm.
+ c.updateResourceVersion(event, state)
+ state.resetBackoff()
+ default:
+ resp := c.kserviceEventToWatchResponse(ctx, event)
+ if resp == nil {
+ continue
+ }
+ select {
+ case ch <- resp:
+ // Advance RV only after confirmed delivery so a failed send
+ // doesn't silently skip the event on the next reconnect.
+ c.updateResourceVersion(event, state)
+ state.resetBackoff()
+ case <-ctx.Done():
+ return false, false
+ }
+ }
+ }
+ }
+}
+
+// updateResourceVersion extracts and stores the latest resourceVersion from a watch event.
+// For Bookmark events it is called immediately; for data events only after successful delivery.
+func (c *AppK8sClient) updateResourceVersion(event k8swatch.Event, state *watchState) {
+ switch event.Type {
+ case k8swatch.Added, k8swatch.Modified, k8swatch.Deleted, k8swatch.Bookmark:
+ if ksvc, ok := event.Object.(*servingv1.Service); ok {
+ if rv := ksvc.GetResourceVersion(); rv != "" {
+ state.lastResourceVersion = rv
+ }
+ }
+ }
+}
+
+// kserviceEventToWatchResponse maps a K8s watch event to a flyteapp.WatchResponse.
+// Returns nil for event types that should not be forwarded (Error, Bookmark).
+func (c *AppK8sClient) kserviceEventToWatchResponse(ctx context.Context, event k8swatch.Event) *flyteapp.WatchResponse {
+ ksvc, ok := event.Object.(*servingv1.Service)
+ if !ok {
+ return nil
+ }
+ app, err := c.kserviceToApp(ctx, ksvc)
+ if err != nil {
+ // KService is not managed by us — skip it.
+ return nil
+ }
+ switch event.Type {
+ case k8swatch.Added:
+ return &flyteapp.WatchResponse{
+ Event: &flyteapp.WatchResponse_CreateEvent{
+ CreateEvent: &flyteapp.CreateEvent{App: app},
+ },
+ }
+ case k8swatch.Modified:
+ return &flyteapp.WatchResponse{
+ Event: &flyteapp.WatchResponse_UpdateEvent{
+ UpdateEvent: &flyteapp.UpdateEvent{UpdatedApp: app},
+ },
+ }
+ case k8swatch.Deleted:
+ return &flyteapp.WatchResponse{
+ Event: &flyteapp.WatchResponse_DeleteEvent{
+ DeleteEvent: &flyteapp.DeleteEvent{App: app},
+ },
+ }
+ default:
+ return nil
+ }
+}
+
+// GetStatus reads the KService and maps its conditions to a flyteapp.Status proto.
+func (c *AppK8sClient) GetStatus(ctx context.Context, appID *flyteapp.Identifier) (*flyteapp.Status, error) {
+ ns := appNamespace(appID.GetProject(), appID.GetDomain())
+ name := kserviceName(appID)
+ ksvc := &servingv1.Service{}
+ if err := c.k8sClient.Get(ctx, client.ObjectKey{Name: name, Namespace: ns}, ksvc); err != nil {
+ if k8serrors.IsNotFound(err) {
+ return nil, fmt.Errorf("KService %s not found: %w", name, err)
+ }
+ return nil, fmt.Errorf("failed to get KService %s: %w", name, err)
+ }
+ return c.kserviceToStatus(ctx, ksvc), nil
+}
+
+// List returns apps for the given project/domain scope with optional pagination.
+func (c *AppK8sClient) List(ctx context.Context, project, domain, appName string, limit uint32, token string) ([]*flyteapp.App, string, error) {
+ ns := appNamespace(project, domain)
+
+ matchLabels := client.MatchingLabels{labelAppManaged: "true"}
+ if appName != "" {
+ matchLabels[labelAppName] = strings.ToLower(appName)
+ }
+ listOpts := []client.ListOption{
+ client.InNamespace(ns),
+ matchLabels,
+ }
+ if limit > 0 {
+ listOpts = append(listOpts, client.Limit(int64(limit)))
+ }
+ if token != "" {
+ listOpts = append(listOpts, client.Continue(token))
+ }
+
+ list := &servingv1.ServiceList{}
+ if err := c.k8sClient.List(ctx, list, listOpts...); err != nil {
+ return nil, "", fmt.Errorf("failed to list KServices for %s/%s: %w", project, domain, err)
+ }
+
+ apps := make([]*flyteapp.App, 0, len(list.Items))
+ for i := range list.Items {
+ a, err := c.kserviceToApp(ctx, &list.Items[i])
+ if err != nil {
+ logger.Warnf(ctx, "Skipping KService %s: failed to convert to app: %v", list.Items[i].Name, err)
+ continue
+ }
+ apps = append(apps, a)
+ }
+ return apps, list.Continue, nil
+}
+
+// --- Helpers ---
+
+// kserviceName returns the KService name for an app. Since each app is deployed
+// to its own project/domain namespace, the name only needs to be unique within
+// that namespace — the app name alone suffices.
+// Names are lower-cased and capped at 63 chars (K8s DNS label limit). For names
+// that exceed 63 chars, the first 54 chars are kept and an 8-char SHA256 suffix
+// is appended to avoid collisions between names with a long common prefix.
+func kserviceName(id *flyteapp.Identifier) string {
+ name := strings.ToLower(id.GetName())
+ if len(name) <= maxKServiceNameLen {
+ return name
+ }
+ sum := sha256.Sum256([]byte(name))
+ suffix := hex.EncodeToString(sum[:4]) // 4 bytes = 8 hex chars
+ return name[:maxKServiceNameLen-9] + "-" + suffix
+}
+
+// specSHA computes a SHA256 digest of the serialized App Spec proto.
+func specSHA(spec *flyteapp.Spec) (string, error) {
+ b, err := proto.Marshal(spec)
+ if err != nil {
+ return "", fmt.Errorf("failed to marshal spec: %w", err)
+ }
+ sum := sha256.Sum256(b)
+ return hex.EncodeToString(sum[:8]), nil // 8 bytes = 16 hex chars, enough for change detection
+}
+
+// buildKService constructs a Knative Service manifest from an App proto.
+func (c *AppK8sClient) buildKService(app *flyteapp.App) (*servingv1.Service, error) {
+ appID := app.GetMetadata().GetId()
+ spec := app.GetSpec()
+ name := kserviceName(appID)
+ ns := appNamespace(appID.GetProject(), appID.GetDomain())
+
+ sha, err := specSHA(spec)
+ if err != nil {
+ return nil, err
+ }
+
+ podSpec, err := buildPodSpec(spec)
+ if err != nil {
+ return nil, err
+ }
+
+ templateAnnotations := buildAutoscalingAnnotations(spec, c.cfg)
+
+ timeoutSecs := c.cfg.DefaultRequestTimeout.Seconds()
+ if t := spec.GetTimeouts().GetRequestTimeout(); t != nil {
+ timeoutSecs = t.AsDuration().Seconds()
+ if timeoutSecs > c.cfg.MaxRequestTimeout.Seconds() {
+ timeoutSecs = c.cfg.MaxRequestTimeout.Seconds()
+ }
+ }
+ timeoutSecsInt := int64(timeoutSecs)
+
+ ksvc := &servingv1.Service{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: ns,
+ Labels: map[string]string{
+ labelAppManaged: "true",
+ labelProject: appID.GetProject(),
+ labelDomain: appID.GetDomain(),
+ labelAppName: appID.GetName(),
+ },
+ Annotations: map[string]string{
+ annotationSpecSHA: sha,
+ annotationAppID: fmt.Sprintf("%s/%s/%s", appID.GetProject(), appID.GetDomain(), appID.GetName()),
+ },
+ },
+ Spec: servingv1.ServiceSpec{
+ ConfigurationSpec: servingv1.ConfigurationSpec{
+ Template: servingv1.RevisionTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: templateAnnotations,
+ },
+ Spec: servingv1.RevisionSpec{
+ PodSpec: podSpec,
+ TimeoutSeconds: &timeoutSecsInt,
+ },
+ },
+ },
+ },
+ }
+ return ksvc, nil
+}
+
+// buildPodSpec constructs a corev1.PodSpec from an App Spec.
+// Supports Container payload only for now; K8sPod support can be added in a follow-up.
+func buildPodSpec(spec *flyteapp.Spec) (corev1.PodSpec, error) {
+ switch p := spec.GetAppPayload().(type) {
+ case *flyteapp.Spec_Container:
+ c := p.Container
+ container := corev1.Container{
+ Name: "app",
+ Image: c.GetImage(),
+ Args: c.GetArgs(),
+ }
+ for _, e := range c.GetEnv() {
+ container.Env = append(container.Env, corev1.EnvVar{
+ Name: e.GetKey(),
+ Value: e.GetValue(),
+ })
+ }
+ return corev1.PodSpec{Containers: []corev1.Container{container}}, nil
+
+ case *flyteapp.Spec_Pod:
+ // K8sPod payloads are not yet supported — the pod spec serialization
+ // from flyteplugins is needed for a complete implementation.
+ return corev1.PodSpec{}, fmt.Errorf("K8sPod app payload is not yet supported")
+
+ default:
+ return corev1.PodSpec{}, fmt.Errorf("app spec has no payload (container or pod required)")
+ }
+}
+
+// buildAutoscalingAnnotations returns the Knative autoscaling annotations for the revision template.
+func buildAutoscalingAnnotations(spec *flyteapp.Spec, cfg *config.InternalAppConfig) map[string]string {
+ annotations := map[string]string{}
+ autoscaling := spec.GetAutoscaling()
+ if autoscaling == nil {
+ return annotations
+ }
+
+ if r := autoscaling.GetReplicas(); r != nil {
+ annotations["autoscaling.knative.dev/min-scale"] = fmt.Sprintf("%d", r.GetMin())
+ annotations["autoscaling.knative.dev/max-scale"] = fmt.Sprintf("%d", r.GetMax())
+ }
+
+ if m := autoscaling.GetScalingMetric(); m != nil {
+ switch metric := m.GetMetric().(type) {
+ case *flyteapp.ScalingMetric_RequestRate:
+ annotations["autoscaling.knative.dev/metric"] = "rps"
+ annotations["autoscaling.knative.dev/target"] = fmt.Sprintf("%d", metric.RequestRate.GetTargetValue())
+ case *flyteapp.ScalingMetric_Concurrency:
+ annotations["autoscaling.knative.dev/metric"] = "concurrency"
+ annotations["autoscaling.knative.dev/target"] = fmt.Sprintf("%d", metric.Concurrency.GetTargetValue())
+ }
+ }
+
+ if p := autoscaling.GetScaledownPeriod(); p != nil {
+ annotations["autoscaling.knative.dev/window"] = p.AsDuration().String()
+ }
+
+ return annotations
+}
+
+// statusWithPhase builds a flyteapp.Status with a single Condition set to the given phase.
+func statusWithPhase(phase flyteapp.Status_DeploymentStatus, message string) *flyteapp.Status {
+ return &flyteapp.Status{
+ Conditions: []*flyteapp.Condition{
+ {
+ DeploymentStatus: phase,
+ Message: message,
+ LastTransitionTime: timestamppb.Now(),
+ },
+ },
+ }
+}
+
+// kserviceToStatus maps a KService's conditions to a flyteapp.Status proto.
+// It fetches the latest ready Revision to read the accurate ActualReplicas count.
+func (c *AppK8sClient) kserviceToStatus(ctx context.Context, ksvc *servingv1.Service) *flyteapp.Status {
+ var phase flyteapp.Status_DeploymentStatus
+ var message string
+
+ // Check if max-scale=0 is set — explicitly stopped by the control plane.
+ if ann := ksvc.Spec.Template.Annotations; ann != nil {
+ if ann["autoscaling.knative.dev/max-scale"] == maxScaleZero {
+ phase = flyteapp.Status_DEPLOYMENT_STATUS_STOPPED
+ message = "App scaled to zero"
+ }
+ }
+
+ if phase == flyteapp.Status_DEPLOYMENT_STATUS_UNSPECIFIED {
+ switch {
+ case ksvc.IsReady():
+ phase = flyteapp.Status_DEPLOYMENT_STATUS_ACTIVE
+ case ksvc.IsFailed():
+ phase = flyteapp.Status_DEPLOYMENT_STATUS_FAILED
+ if c := ksvc.Status.GetCondition(servingv1.ServiceConditionReady); c != nil {
+ message = c.Message
+ }
+ case ksvc.Status.LatestCreatedRevisionName != ksvc.Status.LatestReadyRevisionName:
+ phase = flyteapp.Status_DEPLOYMENT_STATUS_DEPLOYING
+ default:
+ phase = flyteapp.Status_DEPLOYMENT_STATUS_PENDING
+ }
+ }
+
+ status := statusWithPhase(phase, message)
+
+ // Populate ingress URL from KService route status.
+ if url := ksvc.Status.URL; url != nil {
+ status.Ingress = &flyteapp.Ingress{
+ PublicUrl: url.String(),
+ }
+ }
+
+ // Populate current replica count from the latest ready Revision.
+ if revName := ksvc.Status.LatestReadyRevisionName; revName != "" {
+ rev := &servingv1.Revision{}
+ if err := c.k8sClient.Get(ctx, client.ObjectKey{Name: revName, Namespace: ksvc.Namespace}, rev); err == nil {
+ if rev.Status.ActualReplicas != nil {
+ status.CurrentReplicas = uint32(*rev.Status.ActualReplicas)
+ }
+ }
+ }
+ status.K8SMetadata = &flyteapp.K8SMetadata{
+ Namespace: ksvc.Namespace,
+ }
+
+ return status
+}
+
+// GetReplicas lists the pods currently backing the given app.
+func (c *AppK8sClient) GetReplicas(ctx context.Context, appID *flyteapp.Identifier) ([]*flyteapp.Replica, error) {
+ ns := appNamespace(appID.GetProject(), appID.GetDomain())
+ podList := &corev1.PodList{}
+ if err := c.k8sClient.List(ctx, podList,
+ client.InNamespace(ns),
+ client.MatchingLabels{labelAppName: appID.GetName()},
+ ); err != nil {
+ return nil, fmt.Errorf("failed to list pods for app %s/%s/%s: %w",
+ appID.GetProject(), appID.GetDomain(), appID.GetName(), err)
+ }
+
+ replicas := make([]*flyteapp.Replica, 0, len(podList.Items))
+ for i := range podList.Items {
+ replicas = append(replicas, podToReplica(appID, &podList.Items[i]))
+ }
+ return replicas, nil
+}
+
+// DeleteReplica force-deletes a specific pod. Knative will schedule a replacement automatically.
+func (c *AppK8sClient) DeleteReplica(ctx context.Context, replicaID *flyteapp.ReplicaIdentifier) error {
+ appID := replicaID.GetAppId()
+ ns := appNamespace(appID.GetProject(), appID.GetDomain())
+ pod := &corev1.Pod{}
+ pod.Name = replicaID.GetName()
+ pod.Namespace = ns
+ if err := c.k8sClient.Delete(ctx, pod); err != nil {
+ if k8serrors.IsNotFound(err) {
+ return nil
+ }
+ return fmt.Errorf("failed to delete pod %s/%s: %w", ns, replicaID.GetName(), err)
+ }
+ logger.Infof(ctx, "Deleted replica pod %s/%s", ns, replicaID.GetName())
+ return nil
+}
+
+// podToReplica maps a corev1.Pod to a flyteapp.Replica proto.
+func podToReplica(appID *flyteapp.Identifier, pod *corev1.Pod) *flyteapp.Replica {
+ status, reason := podDeploymentStatus(pod)
+ return &flyteapp.Replica{
+ Metadata: &flyteapp.ReplicaMeta{
+ Id: &flyteapp.ReplicaIdentifier{
+ AppId: appID,
+ Name: pod.Name,
+ },
+ },
+ Status: &flyteapp.ReplicaStatus{
+ DeploymentStatus: status,
+ Reason: reason,
+ },
+ }
+}
+
+// podDeploymentStatus maps a pod's phase and conditions to a status string and reason.
+func podDeploymentStatus(pod *corev1.Pod) (string, string) {
+ switch pod.Status.Phase {
+ case corev1.PodRunning:
+ for _, cs := range pod.Status.ContainerStatuses {
+ if !cs.Ready {
+ if cs.State.Waiting != nil {
+ return "DEPLOYING", cs.State.Waiting.Reason
+ }
+ return "DEPLOYING", "container not ready"
+ }
+ }
+ return "ACTIVE", ""
+ case corev1.PodPending:
+ for _, cs := range pod.Status.ContainerStatuses {
+ if cs.State.Waiting != nil && cs.State.Waiting.Reason != "" {
+ return "PENDING", cs.State.Waiting.Reason
+ }
+ }
+ return "PENDING", string(pod.Status.Phase)
+ case corev1.PodFailed:
+ reason := pod.Status.Reason
+ if reason == "" && len(pod.Status.ContainerStatuses) > 0 {
+ if t := pod.Status.ContainerStatuses[0].State.Terminated; t != nil {
+ reason = t.Reason
+ }
+ }
+ return "FAILED", reason
+ case corev1.PodSucceeded:
+ return "STOPPED", "pod completed"
+ default:
+ return "PENDING", string(pod.Status.Phase)
+ }
+}
+
+// kserviceToApp reconstructs a flyteapp.App from a KService by reading the
+// app identifier from annotations and the live status from KService conditions.
+func (c *AppK8sClient) kserviceToApp(ctx context.Context, ksvc *servingv1.Service) (*flyteapp.App, error) {
+ appIDStr, ok := ksvc.Annotations[annotationAppID]
+ if !ok {
+ return nil, fmt.Errorf("KService %s missing %s annotation", ksvc.Name, annotationAppID)
+ }
+
+ // annotation format: "{project}/{domain}/{name}"
+ parts := strings.SplitN(appIDStr, "/", 3)
+ if len(parts) != 3 {
+ return nil, fmt.Errorf("KService %s has malformed %s annotation: %q", ksvc.Name, annotationAppID, appIDStr)
+ }
+
+ appID := &flyteapp.Identifier{
+ Project: parts[0],
+ Domain: parts[1],
+ Name: parts[2],
+ }
+
+ return &flyteapp.App{
+ Metadata: &flyteapp.Meta{
+ Id: appID,
+ },
+ Status: c.kserviceToStatus(ctx, ksvc),
+ }, nil
+}
diff --git a/app/internal/k8s/app_client_test.go b/app/internal/k8s/app_client_test.go
new file mode 100644
index 00000000000..cb04428a297
--- /dev/null
+++ b/app/internal/k8s/app_client_test.go
@@ -0,0 +1,791 @@
+package k8s
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ corev1 "k8s.io/api/core/v1"
+ k8serrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ k8swatch "k8s.io/apimachinery/pkg/watch"
+ servingv1 "knative.dev/serving/pkg/apis/serving/v1"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+ "github.com/flyteorg/flyte/v2/app/internal/config"
+ flyteapp "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app"
+ flytecoreapp "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+// testScheme builds a runtime.Scheme with Knative and core types registered.
+func testScheme(t *testing.T) *runtime.Scheme {
+ t.Helper()
+ s := runtime.NewScheme()
+ require.NoError(t, corev1.AddToScheme(s))
+ require.NoError(t, servingv1.AddToScheme(s))
+ return s
+}
+
+// testRevision builds a Knative Revision object with a given ActualReplicas count.
+func testRevision(name, namespace string, actualReplicas int32) *servingv1.Revision {
+ return &servingv1.Revision{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: namespace,
+ },
+ Status: servingv1.RevisionStatus{
+ ActualReplicas: &actualReplicas,
+ },
+ }
+}
+
+// testClient builds an AppK8sClient backed by a fake K8s client.
+func testClient(t *testing.T, objs ...client.Object) *AppK8sClient {
+ t.Helper()
+ s := testScheme(t)
+ fc := fake.NewClientBuilder().
+ WithScheme(s).
+ WithObjects(objs...).
+ Build()
+ return &AppK8sClient{
+ k8sClient: fc,
+ cfg: &config.InternalAppConfig{
+ DefaultRequestTimeout: 5 * time.Minute,
+ MaxRequestTimeout: time.Hour,
+ },
+ }
+}
+
+// testApp builds a minimal flyteapp.App for use in tests.
+func testApp(project, domain, name, image string) *flyteapp.App {
+ return &flyteapp.App{
+ Metadata: &flyteapp.Meta{
+ Id: &flyteapp.Identifier{
+ Project: project,
+ Domain: domain,
+ Name: name,
+ },
+ },
+ Spec: &flyteapp.Spec{
+ AppPayload: &flyteapp.Spec_Container{
+ Container: &flytecoreapp.Container{
+ Image: image,
+ },
+ },
+ },
+ }
+}
+
+func TestDeploy_Create(t *testing.T) {
+ c := testClient(t)
+ app := testApp("proj", "dev", "myapp", "nginx:latest")
+
+ err := c.Deploy(context.Background(), app)
+ require.NoError(t, err)
+
+ ksvc := &servingv1.Service{}
+ err = c.k8sClient.Get(context.Background(),
+ client.ObjectKey{Name: "myapp", Namespace: "proj-dev"}, ksvc)
+ require.NoError(t, err)
+ assert.Equal(t, "proj", ksvc.Labels[labelProject])
+ assert.Equal(t, "dev", ksvc.Labels[labelDomain])
+ assert.Equal(t, "myapp", ksvc.Labels[labelAppName])
+ assert.NotEmpty(t, ksvc.Annotations[annotationSpecSHA])
+ assert.Equal(t, "proj/dev/myapp", ksvc.Annotations[annotationAppID])
+}
+
+func TestDeploy_UpdateOnSpecChange(t *testing.T) {
+ c := testClient(t)
+ app := testApp("proj", "dev", "myapp", "nginx:1.0")
+ require.NoError(t, c.Deploy(context.Background(), app))
+
+ // Change image — spec SHA changes → update should happen.
+ app.Spec.GetContainer().Image = "nginx:2.0"
+ require.NoError(t, c.Deploy(context.Background(), app))
+
+ ksvc := &servingv1.Service{}
+ require.NoError(t, c.k8sClient.Get(context.Background(),
+ client.ObjectKey{Name: "myapp", Namespace: "proj-dev"}, ksvc))
+ assert.Equal(t, "nginx:2.0", ksvc.Spec.Template.Spec.Containers[0].Image)
+}
+
+func TestDeploy_SkipUpdateWhenUnchanged(t *testing.T) {
+ c := testClient(t)
+ app := testApp("proj", "dev", "myapp", "nginx:latest")
+ require.NoError(t, c.Deploy(context.Background(), app))
+
+ // Get initial resource version.
+ ksvc := &servingv1.Service{}
+ require.NoError(t, c.k8sClient.Get(context.Background(),
+ client.ObjectKey{Name: "myapp", Namespace: "proj-dev"}, ksvc))
+ initialRV := ksvc.ResourceVersion
+
+ // Deploy same spec — should be a no-op.
+ require.NoError(t, c.Deploy(context.Background(), app))
+
+ require.NoError(t, c.k8sClient.Get(context.Background(),
+ client.ObjectKey{Name: "myapp", Namespace: "proj-dev"}, ksvc))
+ assert.Equal(t, initialRV, ksvc.ResourceVersion, "resource version should not change on no-op deploy")
+}
+
+func TestStop(t *testing.T) {
+ c := testClient(t)
+ app := testApp("proj", "dev", "myapp", "nginx:latest")
+ require.NoError(t, c.Deploy(context.Background(), app))
+
+ id := &flyteapp.Identifier{Project: "proj", Domain: "dev", Name: "myapp"}
+ require.NoError(t, c.Stop(context.Background(), id))
+
+ ksvc := &servingv1.Service{}
+ require.NoError(t, c.k8sClient.Get(context.Background(),
+ client.ObjectKey{Name: "myapp", Namespace: "proj-dev"}, ksvc))
+ assert.Equal(t, "0", ksvc.Spec.Template.Annotations["autoscaling.knative.dev/max-scale"])
+}
+
+func TestStop_NotFound(t *testing.T) {
+ c := testClient(t)
+ id := &flyteapp.Identifier{Project: "proj", Domain: "dev", Name: "missing"}
+ // Should succeed silently — already gone.
+ require.NoError(t, c.Stop(context.Background(), id))
+}
+
+func TestDelete(t *testing.T) {
+ c := testClient(t)
+ app := testApp("proj", "dev", "myapp", "nginx:latest")
+ require.NoError(t, c.Deploy(context.Background(), app))
+
+ id := &flyteapp.Identifier{Project: "proj", Domain: "dev", Name: "myapp"}
+ require.NoError(t, c.Delete(context.Background(), id))
+
+ ksvc := &servingv1.Service{}
+ err := c.k8sClient.Get(context.Background(),
+ client.ObjectKey{Name: "myapp", Namespace: "proj-dev"}, ksvc)
+ assert.True(t, k8serrors.IsNotFound(err))
+}
+
+func TestDelete_NotFound(t *testing.T) {
+ c := testClient(t)
+ id := &flyteapp.Identifier{Project: "proj", Domain: "dev", Name: "missing"}
+ require.NoError(t, c.Delete(context.Background(), id))
+}
+
+func TestGetStatus_NotFound(t *testing.T) {
+ c := testClient(t)
+ id := &flyteapp.Identifier{Project: "proj", Domain: "dev", Name: "missing"}
+ status, err := c.GetStatus(context.Background(), id)
+ require.Error(t, err)
+ assert.True(t, k8serrors.IsNotFound(err))
+ assert.Nil(t, status)
+}
+
+func TestGetStatus_Stopped(t *testing.T) {
+ c := testClient(t)
+ app := testApp("proj", "dev", "myapp", "nginx:latest")
+ require.NoError(t, c.Deploy(context.Background(), app))
+
+ id := &flyteapp.Identifier{Project: "proj", Domain: "dev", Name: "myapp"}
+ require.NoError(t, c.Stop(context.Background(), id))
+
+ status, err := c.GetStatus(context.Background(), id)
+ require.NoError(t, err)
+ require.Len(t, status.Conditions, 1)
+ assert.Equal(t, flyteapp.Status_DEPLOYMENT_STATUS_STOPPED, status.Conditions[0].DeploymentStatus)
+}
+
+func TestGetStatus_CurrentReplicas(t *testing.T) {
+ s := testScheme(t)
+ // Pre-populate a KService with LatestReadyRevisionName already set in status,
+ // and the corresponding Revision with ActualReplicas=4.
+ ksvc := &servingv1.Service{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "myapp",
+ Namespace: "proj-dev",
+ Labels: map[string]string{
+ labelAppManaged: "true",
+ labelProject: "proj",
+ labelDomain: "dev",
+ labelAppName: "myapp",
+ },
+ Annotations: map[string]string{
+ annotationAppID: "proj/dev/myapp",
+ },
+ },
+ }
+ ksvc.Status.LatestReadyRevisionName = "myapp-00001"
+
+ rev := testRevision("myapp-00001", "proj-dev", 4)
+
+ fc := fake.NewClientBuilder().
+ WithScheme(s).
+ WithObjects(ksvc, rev).
+ WithStatusSubresource(ksvc).
+ Build()
+ c := &AppK8sClient{
+ k8sClient: fc,
+ cfg: &config.InternalAppConfig{},
+ }
+
+ id := &flyteapp.Identifier{Project: "proj", Domain: "dev", Name: "myapp"}
+ status, err := c.GetStatus(context.Background(), id)
+ require.NoError(t, err)
+ assert.Equal(t, uint32(4), status.CurrentReplicas)
+}
+
+func TestList(t *testing.T) {
+ s := testScheme(t)
+ // Pre-populate two KServices with different project labels.
+ ksvc1 := &servingv1.Service{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "app1",
+ Namespace: "proj-dev",
+ Labels: map[string]string{
+ labelAppManaged: "true",
+ labelProject: "proj",
+ labelDomain: "dev",
+ labelAppName: "app1",
+ },
+ Annotations: map[string]string{
+ annotationAppID: "proj/dev/app1",
+ },
+ },
+ }
+ ksvc2 := &servingv1.Service{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "app2",
+ Namespace: "other-dev",
+ Labels: map[string]string{
+ labelAppManaged: "true",
+ labelProject: "other",
+ labelDomain: "dev",
+ labelAppName: "app2",
+ },
+ Annotations: map[string]string{
+ annotationAppID: "other/dev/app2",
+ },
+ },
+ }
+
+ fc := fake.NewClientBuilder().
+ WithScheme(s).
+ WithObjects(ksvc1, ksvc2).
+ Build()
+ c := &AppK8sClient{
+ k8sClient: fc,
+ cfg: &config.InternalAppConfig{
+ DefaultRequestTimeout: 5 * time.Minute,
+ MaxRequestTimeout: time.Hour,
+ },
+ }
+
+ apps, nextToken, err := c.List(context.Background(), "proj", "dev", "", 0, "")
+ require.NoError(t, err)
+ assert.Empty(t, nextToken)
+ require.Len(t, apps, 1)
+ assert.Equal(t, "proj", apps[0].Metadata.Id.Project)
+ assert.Equal(t, "app1", apps[0].Metadata.Id.Name)
+}
+
+func TestGetReplicas(t *testing.T) {
+ s := testScheme(t)
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "myapp-abc",
+ Namespace: "proj-dev",
+ Labels: map[string]string{
+ labelAppName: "myapp",
+ },
+ },
+ Status: corev1.PodStatus{
+ Phase: corev1.PodRunning,
+ ContainerStatuses: []corev1.ContainerStatus{
+ {Ready: true},
+ },
+ },
+ }
+ fc := fake.NewClientBuilder().WithScheme(s).WithObjects(pod).Build()
+ c := &AppK8sClient{
+ k8sClient: fc,
+ cfg: &config.InternalAppConfig{},
+ }
+
+ id := &flyteapp.Identifier{Project: "proj", Domain: "dev", Name: "myapp"}
+ replicas, err := c.GetReplicas(context.Background(), id)
+ require.NoError(t, err)
+ require.Len(t, replicas, 1)
+ assert.Equal(t, "myapp-abc", replicas[0].Metadata.Id.Name)
+ assert.Equal(t, "ACTIVE", replicas[0].Status.DeploymentStatus)
+}
+
+func TestDeleteReplica(t *testing.T) {
+ s := testScheme(t)
+ pod := &corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "myapp-abc",
+ Namespace: "proj-dev",
+ },
+ }
+ fc := fake.NewClientBuilder().WithScheme(s).WithObjects(pod).Build()
+ c := &AppK8sClient{
+ k8sClient: fc,
+ cfg: &config.InternalAppConfig{},
+ }
+
+ replicaID := &flyteapp.ReplicaIdentifier{
+ AppId: &flyteapp.Identifier{Project: "proj", Domain: "dev", Name: "myapp"},
+ Name: "myapp-abc",
+ }
+ require.NoError(t, c.DeleteReplica(context.Background(), replicaID))
+
+ err := fc.Get(context.Background(),
+ client.ObjectKey{Name: "myapp-abc", Namespace: "proj-dev"}, &corev1.Pod{})
+ assert.True(t, k8serrors.IsNotFound(err))
+}
+
+func TestKserviceEventToWatchResponse(t *testing.T) {
+ ksvc := &servingv1.Service{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "myapp",
+ Namespace: "proj-dev",
+ Annotations: map[string]string{
+ annotationAppID: "proj/dev/myapp",
+ },
+ },
+ }
+
+ tests := []struct {
+ eventType k8swatch.EventType
+ wantNil bool
+ wantEventKey string
+ }{
+ {k8swatch.Added, false, "create"},
+ {k8swatch.Modified, false, "update"},
+ {k8swatch.Deleted, false, "delete"},
+ {k8swatch.Error, true, ""},
+ {k8swatch.Bookmark, true, ""},
+ }
+
+ c := testClient(t)
+ for _, tt := range tests {
+ t.Run(string(tt.eventType), func(t *testing.T) {
+ resp := c.kserviceEventToWatchResponse(context.Background(), k8swatch.Event{
+ Type: tt.eventType,
+ Object: ksvc,
+ })
+ if tt.wantNil {
+ assert.Nil(t, resp)
+ return
+ }
+ require.NotNil(t, resp)
+ switch tt.wantEventKey {
+ case "create":
+ assert.NotNil(t, resp.GetCreateEvent())
+ assert.Equal(t, "proj", resp.GetCreateEvent().App.Metadata.Id.Project)
+ case "update":
+ assert.NotNil(t, resp.GetUpdateEvent())
+ assert.Equal(t, "myapp", resp.GetUpdateEvent().UpdatedApp.Metadata.Id.Name)
+ case "delete":
+ assert.NotNil(t, resp.GetDeleteEvent())
+ }
+ })
+ }
+}
+
+func TestKserviceName(t *testing.T) {
+ tests := []struct {
+ name string
+ want string
+ }{
+ {"myapp", "myapp"},
+ {"MyApp", "myapp"},
+ // v1 and v2 variants stay distinct — no truncation collision.
+ {"my-long-service-name-v1", "my-long-service-name-v1"},
+ {"my-long-service-name-v2", "my-long-service-name-v2"},
+ // Names over 63 chars get a hash suffix instead of blind truncation.
+ {
+ "this-is-a-very-long-app-name-that-exceeds-the-kubernetes-dns-label-limit",
+ func() string {
+ name := "this-is-a-very-long-app-name-that-exceeds-the-kubernetes-dns-label-limit"
+ sum := sha256.Sum256([]byte(name))
+ return name[:54] + "-" + hex.EncodeToString(sum[:4])
+ }(),
+ },
+ }
+ for _, tt := range tests {
+ id := &flyteapp.Identifier{Project: "proj", Domain: "dev", Name: tt.name}
+ got := kserviceName(id)
+ assert.Equal(t, tt.want, got)
+ assert.LessOrEqual(t, len(got), maxKServiceNameLen)
+ }
+}
+
+func TestPodDeploymentStatus(t *testing.T) {
+ tests := []struct {
+ name string
+ pod corev1.Pod
+ wantStatus string
+ wantReason string
+ }{
+ {
+ name: "running and ready",
+ pod: corev1.Pod{
+ Status: corev1.PodStatus{
+ Phase: corev1.PodRunning,
+ ContainerStatuses: []corev1.ContainerStatus{{Ready: true}},
+ },
+ },
+ wantStatus: "ACTIVE",
+ },
+ {
+ name: "running but container not ready",
+ pod: corev1.Pod{
+ Status: corev1.PodStatus{
+ Phase: corev1.PodRunning,
+ ContainerStatuses: []corev1.ContainerStatus{
+ {Ready: false, State: corev1.ContainerState{
+ Waiting: &corev1.ContainerStateWaiting{Reason: "ContainerCreating"},
+ }},
+ },
+ },
+ },
+ wantStatus: "DEPLOYING",
+ wantReason: "ContainerCreating",
+ },
+ {
+ name: "pending with waiting reason",
+ pod: corev1.Pod{
+ Status: corev1.PodStatus{
+ Phase: corev1.PodPending,
+ ContainerStatuses: []corev1.ContainerStatus{
+ {State: corev1.ContainerState{
+ Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff"},
+ }},
+ },
+ },
+ },
+ wantStatus: "PENDING",
+ wantReason: "ImagePullBackOff",
+ },
+ {
+ name: "failed",
+ pod: corev1.Pod{
+ Status: corev1.PodStatus{
+ Phase: corev1.PodFailed,
+ Reason: "OOMKilled",
+ },
+ },
+ wantStatus: "FAILED",
+ wantReason: "OOMKilled",
+ },
+ {
+ name: "succeeded",
+ pod: corev1.Pod{
+ Status: corev1.PodStatus{Phase: corev1.PodSucceeded},
+ },
+ wantStatus: "STOPPED",
+ wantReason: "pod completed",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ status, reason := podDeploymentStatus(&tt.pod)
+ assert.Equal(t, tt.wantStatus, status)
+ assert.Equal(t, tt.wantReason, reason)
+ })
+ }
+}
+
+// --- Watch reconnect tests ---
+
+// watchCall is a pre-programmed response for one call to multiWatchClient.Watch.
+type watchCall struct {
+ watcher k8swatch.Interface
+ err error
+}
+
+// multiWatchClient wraps the fake client but intercepts Watch() calls, returning
+// pre-programmed watchers in sequence. All other methods delegate to the embedded fake.
+type multiWatchClient struct {
+ client.WithWatch
+ mu sync.Mutex
+ calls []watchCall
+ callIdx int
+ // capturedRVs records the ResourceVersion passed to each Watch() call (for assertions).
+ capturedRVs []string
+}
+
+func (m *multiWatchClient) Watch(ctx context.Context, list client.ObjectList, opts ...client.ListOption) (k8swatch.Interface, error) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+
+ // Extract ResourceVersion from Raw options if present.
+ lo := &client.ListOptions{}
+ for _, o := range opts {
+ o.ApplyToList(lo)
+ }
+ rv := ""
+ if lo.Raw != nil {
+ rv = lo.Raw.ResourceVersion
+ }
+ m.capturedRVs = append(m.capturedRVs, rv)
+
+ if m.callIdx >= len(m.calls) {
+ // No more programmed calls — return a watcher that blocks until ctx is cancelled.
+ return k8swatch.NewFakeWithChanSize(0, false), nil
+ }
+ c := m.calls[m.callIdx]
+ m.callIdx++
+ if c.err != nil {
+ return nil, c.err
+ }
+ return c.watcher, nil
+}
+
+// newMultiClient builds an AppK8sClient backed by a multiWatchClient.
+func newMultiClient(t *testing.T, calls []watchCall, objs ...client.Object) (*AppK8sClient, *multiWatchClient) {
+ t.Helper()
+ s := testScheme(t)
+ base := fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).Build()
+ mwc := &multiWatchClient{WithWatch: base, calls: calls}
+ return &AppK8sClient{
+ k8sClient: mwc,
+ cfg: &config.InternalAppConfig{},
+ }, mwc
+}
+
+// testKsvc builds a minimal KService that kserviceToApp can parse.
+func testKsvc(name, ns, rv string) *servingv1.Service {
+ return &servingv1.Service{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: ns,
+ ResourceVersion: rv,
+ Annotations: map[string]string{annotationAppID: "proj/dev/" + name},
+ Labels: map[string]string{labelAppManaged: "true"},
+ },
+ }
+}
+
+func TestWatch_ReconnectsOnChannelClose(t *testing.T) {
+ watchBackoffInitial = 0
+ t.Cleanup(func() { watchBackoffInitial = 1 * time.Second })
+
+ w1 := k8swatch.NewFake()
+ w2 := k8swatch.NewFake()
+
+ c, _ := newMultiClient(t, []watchCall{{watcher: w1}, {watcher: w2}})
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ ch, err := c.Watch(ctx, "proj", "dev", "")
+ require.NoError(t, err)
+
+ // Send one event on w1 then close it (simulates K8s timeout/disconnect).
+ w1.Add(testKsvc("myapp", "proj-dev", "100"))
+ w1.Stop()
+
+ recv1 := <-ch
+ require.NotNil(t, recv1.GetCreateEvent(), "expected CreateEvent from first watcher")
+
+ // After reconnect, send an event on w2.
+ go func() {
+ time.Sleep(10 * time.Millisecond)
+ w2.Add(testKsvc("myapp", "proj-dev", "200"))
+ }()
+
+ recv2 := <-ch
+ require.NotNil(t, recv2.GetCreateEvent(), "expected CreateEvent from second watcher after reconnect")
+}
+
+func TestWatch_ReconnectsOnErrorEvent(t *testing.T) {
+ watchBackoffInitial = 0
+ t.Cleanup(func() { watchBackoffInitial = 1 * time.Second })
+
+ w1 := k8swatch.NewFake()
+ w2 := k8swatch.NewFake()
+
+ c, _ := newMultiClient(t, []watchCall{{watcher: w1}, {watcher: w2}})
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ ch, err := c.Watch(ctx, "proj", "dev", "")
+ require.NoError(t, err)
+
+ // Send a K8s Error event — should trigger reconnect.
+ w1.Error(&metav1.Status{Code: 410, Reason: metav1.StatusReasonExpired, Message: "too old resource version"})
+
+ go func() {
+ time.Sleep(10 * time.Millisecond)
+ w2.Add(testKsvc("myapp", "proj-dev", "300"))
+ }()
+
+ resp := <-ch
+ require.NotNil(t, resp.GetCreateEvent(), "expected CreateEvent from second watcher after error-triggered reconnect")
+}
+
+func TestWatch_BookmarkUpdatesResourceVersion(t *testing.T) {
+ watchBackoffInitial = 0
+ t.Cleanup(func() { watchBackoffInitial = 1 * time.Second })
+
+ w1 := k8swatch.NewFake()
+ w2 := k8swatch.NewFake()
+
+ c, mwc := newMultiClient(t, []watchCall{{watcher: w1}, {watcher: w2}})
+
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ ch, err := c.Watch(ctx, "proj", "dev", "")
+ require.NoError(t, err)
+
+ // Send a Bookmark with RV=999 then close w1.
+ w1.Action(k8swatch.Bookmark, &servingv1.Service{
+ ObjectMeta: metav1.ObjectMeta{ResourceVersion: "999"},
+ })
+ w1.Stop()
+
+ // Deliver an event from w2 after reconnect.
+ go func() {
+ time.Sleep(10 * time.Millisecond)
+ w2.Add(testKsvc("myapp", "proj-dev", "1000"))
+ }()
+
+ resp := <-ch
+ require.NotNil(t, resp.GetCreateEvent())
+
+ // The second Watch() call should have been made with ResourceVersion="999".
+ mwc.mu.Lock()
+ rvs := append([]string(nil), mwc.capturedRVs...)
+ mwc.mu.Unlock()
+
+ require.GreaterOrEqual(t, len(rvs), 2, "expected at least 2 Watch calls")
+ assert.Equal(t, "", rvs[0], "first Watch call should have no resourceVersion")
+ assert.Equal(t, "999", rvs[1], "second Watch call should use Bookmark resourceVersion")
+}
+
+func TestWatch_ExponentialBackoff(t *testing.T) {
+ watchBackoffInitial = 10 * time.Millisecond
+ watchBackoffMax = 80 * time.Millisecond
+ watchBackoffFactor = 2.0
+ t.Cleanup(func() {
+ watchBackoffInitial = 1 * time.Second
+ watchBackoffMax = 30 * time.Second
+ })
+
+ // Four watchers that each emit an Error event — only Error events trigger backoff.
+ // NewFakeWithChanSize(1,...) gives a buffer of 1 so pre-sends don't block before
+ // the consumer goroutine starts (NewFake() is unbuffered).
+ calls := make([]watchCall, 4)
+ for i := range calls {
+ w := k8swatch.NewFakeWithChanSize(1, false)
+ calls[i] = watchCall{watcher: w}
+ w.Error(&metav1.Status{Code: 410, Reason: metav1.StatusReasonExpired, Message: "resource version too old"})
+ }
+
+ c, mwc := newMultiClient(t, calls)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+
+ start := time.Now()
+ ch, err := c.Watch(ctx, "proj", "dev", "")
+ require.NoError(t, err)
+
+ // Wait until at least 4 Watch() calls have been made.
+ require.Eventually(t, func() bool {
+ mwc.mu.Lock()
+ defer mwc.mu.Unlock()
+ return len(mwc.capturedRVs) >= 4
+ }, 2*time.Second, 5*time.Millisecond)
+
+ elapsed := time.Since(start)
+ // With 10ms+20ms+40ms backoffs before 4th call, minimum elapsed ≈ 70ms.
+ assert.GreaterOrEqual(t, elapsed, 60*time.Millisecond, "backoff should accumulate across error reconnects")
+
+ cancel()
+ for range ch {
+ }
+}
+
+func TestWatch_CleanCloseNoBackoff(t *testing.T) {
+ watchBackoffInitial = 50 * time.Millisecond
+ watchBackoffMax = 200 * time.Millisecond
+ t.Cleanup(func() {
+ watchBackoffInitial = 1 * time.Second
+ watchBackoffMax = 30 * time.Second
+ })
+
+ // Three watchers that close immediately (clean channel close, no Error event).
+ calls := []watchCall{
+ {watcher: k8swatch.NewFake()},
+ {watcher: k8swatch.NewFake()},
+ {watcher: k8swatch.NewFake()},
+ }
+ for _, wc := range calls {
+ wc.watcher.(*k8swatch.FakeWatcher).Stop()
+ }
+
+ c, mwc := newMultiClient(t, calls)
+
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+
+ start := time.Now()
+ ch, err := c.Watch(ctx, "proj", "dev", "")
+ require.NoError(t, err)
+
+ require.Eventually(t, func() bool {
+ mwc.mu.Lock()
+ defer mwc.mu.Unlock()
+ return len(mwc.capturedRVs) >= 3
+ }, 500*time.Millisecond, 5*time.Millisecond)
+
+ elapsed := time.Since(start)
+ // Clean closes must not apply backoff — all 3 reconnects should be nearly instant.
+ assert.Less(t, elapsed, 30*time.Millisecond, "clean closes should not apply backoff delay")
+
+ cancel()
+ for range ch {
+ }
+}
+
+func TestWatch_ContextCancelStopsLoop(t *testing.T) {
+ watchBackoffInitial = 0
+ t.Cleanup(func() { watchBackoffInitial = 1 * time.Second })
+
+ w1 := k8swatch.NewFake()
+ c, _ := newMultiClient(t, []watchCall{{watcher: w1}})
+
+ ctx, cancel := context.WithCancel(context.Background())
+ ch, err := c.Watch(ctx, "proj", "dev", "")
+ require.NoError(t, err)
+
+ cancel()
+
+ select {
+ case _, ok := <-ch:
+ assert.False(t, ok, "channel should be closed after ctx cancel")
+ case <-time.After(500 * time.Millisecond):
+ t.Fatal("channel not closed within 500ms of ctx cancel")
+ }
+}
+
+func TestWatch_InitialWatchErrorReturnsError(t *testing.T) {
+ c, _ := newMultiClient(t, []watchCall{
+ {watcher: nil, err: fmt.Errorf("RBAC denied")},
+ })
+
+ ch, err := c.Watch(context.Background(), "proj", "dev", "")
+ require.Error(t, err)
+ assert.Nil(t, ch)
+}
diff --git a/app/internal/service/internal_app_service.go b/app/internal/service/internal_app_service.go
new file mode 100644
index 00000000000..dc847ec4823
--- /dev/null
+++ b/app/internal/service/internal_app_service.go
@@ -0,0 +1,252 @@
+package service
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ "connectrpc.com/connect"
+ timestamppb "google.golang.org/protobuf/types/known/timestamppb"
+ k8serrors "k8s.io/apimachinery/pkg/api/errors"
+
+ appconfig "github.com/flyteorg/flyte/v2/app/internal/config"
+ appk8s "github.com/flyteorg/flyte/v2/app/internal/k8s"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ flyteapp "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app/appconnect"
+)
+
+// InternalAppService is the data plane implementation of the AppService.
+// It has direct K8s access via AppK8sClientInterface and no database dependency —
+// all app state lives in KService CRDs.
+type InternalAppService struct {
+ appconnect.UnimplementedAppServiceHandler
+ k8s appk8s.AppK8sClientInterface
+ cfg *appconfig.InternalAppConfig
+}
+
+// NewInternalAppService creates a new InternalAppService.
+func NewInternalAppService(k8s appk8s.AppK8sClientInterface, cfg *appconfig.InternalAppConfig) *InternalAppService {
+ return &InternalAppService{k8s: k8s, cfg: cfg}
+}
+
+// Ensure InternalAppService satisfies the generated handler interface.
+var _ appconnect.AppServiceHandler = (*InternalAppService)(nil)
+
+// Create deploys a new app as a KService CRD.
+func (s *InternalAppService) Create(
+ ctx context.Context,
+ req *connect.Request[flyteapp.CreateRequest],
+) (*connect.Response[flyteapp.CreateResponse], error) {
+ app := req.Msg.GetApp()
+ if app.GetMetadata().GetId() == nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("app identifier is required"))
+ }
+ if app.GetSpec() == nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("app spec is required"))
+ }
+ if app.GetSpec().GetAppPayload() == nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("app spec must include a container or pod payload"))
+ }
+
+ if err := s.k8s.Deploy(ctx, app); err != nil {
+ logger.Errorf(ctx, "Failed to deploy app %s: %v", app.GetMetadata().GetId().GetName(), err)
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+
+ app.Status = &flyteapp.Status{
+ Conditions: []*flyteapp.Condition{
+ {
+ DeploymentStatus: flyteapp.Status_DEPLOYMENT_STATUS_PENDING,
+ LastTransitionTime: timestamppb.Now(),
+ },
+ },
+ Ingress: publicIngress(app.GetMetadata().GetId(), s.cfg.BaseDomain),
+ }
+
+ return connect.NewResponse(&flyteapp.CreateResponse{App: app}), nil
+}
+
+// publicIngress builds the deterministic public URL for an app.
+// Pattern: "https://{name}-{project}-{domain}.{base_domain}"
+// Returns nil if BaseDomain is not configured.
+func publicIngress(id *flyteapp.Identifier, baseDomain string) *flyteapp.Ingress {
+ if baseDomain == "" {
+ return nil
+ }
+ host := strings.ToLower(fmt.Sprintf("%s-%s-%s.%s",
+ id.GetName(), id.GetProject(), id.GetDomain(), baseDomain))
+ return &flyteapp.Ingress{
+ PublicUrl: "https://" + host,
+ }
+}
+
+// Get retrieves an app and its live status from the KService CRD.
+// Note: App.Spec is not populated — status and ingress URL are the authoritative fields.
+func (s *InternalAppService) Get(
+ ctx context.Context,
+ req *connect.Request[flyteapp.GetRequest],
+) (*connect.Response[flyteapp.GetResponse], error) {
+ appID, ok := req.Msg.GetIdentifier().(*flyteapp.GetRequest_AppId)
+ if !ok || appID.AppId == nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("app_id is required"))
+ }
+
+ status, err := s.k8s.GetStatus(ctx, appID.AppId)
+ if err != nil {
+ if k8serrors.IsNotFound(err) {
+ return nil, connect.NewError(connect.CodeNotFound, err)
+ }
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+
+ return connect.NewResponse(&flyteapp.GetResponse{
+ App: &flyteapp.App{
+ Metadata: &flyteapp.Meta{Id: appID.AppId},
+ Status: status,
+ },
+ }), nil
+}
+
+// Update modifies an app's spec or desired state.
+// When Spec.DesiredState is STOPPED, the app is scaled to zero (KService kept).
+// When Spec.DesiredState is STARTED or ACTIVE, the app is redeployed/resumed.
+// Otherwise the spec update is applied and the app is redeployed.
+func (s *InternalAppService) Update(
+ ctx context.Context,
+ req *connect.Request[flyteapp.UpdateRequest],
+) (*connect.Response[flyteapp.UpdateResponse], error) {
+ app := req.Msg.GetApp()
+ if app.GetMetadata().GetId() == nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("app identifier is required"))
+ }
+
+ appID := app.GetMetadata().GetId()
+
+ switch app.GetSpec().GetDesiredState() {
+ case flyteapp.Spec_DESIRED_STATE_STOPPED:
+ if err := s.k8s.Stop(ctx, appID); err != nil {
+ logger.Errorf(ctx, "Failed to stop app %s: %v", appID.GetName(), err)
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+ default:
+ // UNSPECIFIED, STARTED, ACTIVE — deploy/redeploy the spec.
+ if err := s.k8s.Deploy(ctx, app); err != nil {
+ logger.Errorf(ctx, "Failed to update app %s: %v", appID.GetName(), err)
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+ }
+
+ status, err := s.k8s.GetStatus(ctx, appID)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+ app.Status = status
+
+ return connect.NewResponse(&flyteapp.UpdateResponse{App: app}), nil
+}
+
+// Delete removes the KService CRD for the given app entirely.
+func (s *InternalAppService) Delete(
+ ctx context.Context,
+ req *connect.Request[flyteapp.DeleteRequest],
+) (*connect.Response[flyteapp.DeleteResponse], error) {
+ appID := req.Msg.GetAppId()
+ if appID == nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("app_id is required"))
+ }
+
+ if err := s.k8s.Delete(ctx, appID); err != nil {
+ logger.Errorf(ctx, "Failed to delete app %s: %v", appID.GetName(), err)
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+
+ return connect.NewResponse(&flyteapp.DeleteResponse{}), nil
+}
+
+// List returns apps for the requested scope with pagination.
+func (s *InternalAppService) List(
+ ctx context.Context,
+ req *connect.Request[flyteapp.ListRequest],
+) (*connect.Response[flyteapp.ListResponse], error) {
+ var project, domain string
+
+ switch f := req.Msg.GetFilterBy().(type) {
+ case *flyteapp.ListRequest_Project:
+ project = f.Project.GetName()
+ domain = f.Project.GetDomain()
+ case *flyteapp.ListRequest_Org, *flyteapp.ListRequest_ClusterId:
+ return nil, connect.NewError(connect.CodeUnimplemented, fmt.Errorf("org and cluster_id filters are not supported by the data plane"))
+ }
+
+ var limit uint32
+ var token string
+ if r := req.Msg.GetRequest(); r != nil {
+ limit = r.GetLimit()
+ token = r.GetToken()
+ }
+
+ apps, nextToken, err := s.k8s.List(ctx, project, domain, "", limit, token)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+
+ return connect.NewResponse(&flyteapp.ListResponse{Apps: apps, Token: nextToken}), nil
+}
+
+// Watch streams live KService events to the client.
+// It first sends the current state as CreateEvents (initial snapshot), then streams changes.
+func (s *InternalAppService) Watch(
+ ctx context.Context,
+ req *connect.Request[flyteapp.WatchRequest],
+ stream *connect.ServerStream[flyteapp.WatchResponse],
+) error {
+ var project, domain, appName string
+
+ switch t := req.Msg.GetTarget().(type) {
+ case *flyteapp.WatchRequest_Project:
+ project = t.Project.GetName()
+ domain = t.Project.GetDomain()
+ case *flyteapp.WatchRequest_AppId:
+ project = t.AppId.GetProject()
+ domain = t.AppId.GetDomain()
+ appName = t.AppId.GetName()
+ case *flyteapp.WatchRequest_Org, *flyteapp.WatchRequest_ClusterId:
+ return connect.NewError(connect.CodeUnimplemented, fmt.Errorf("org and cluster_id watch targets are not supported by the data plane"))
+ }
+
+ // Start watch before listing so no events are lost between the two calls.
+ ch, err := s.k8s.Watch(ctx, project, domain, appName)
+ if err != nil {
+ return connect.NewError(connect.CodeInternal, err)
+ }
+
+ // Send initial snapshot so the client has current state before streaming changes.
+ snapshot, _, err := s.k8s.List(ctx, project, domain, appName, 0, "")
+ if err != nil {
+ return connect.NewError(connect.CodeInternal, err)
+ }
+ for _, app := range snapshot {
+ if err := stream.Send(&flyteapp.WatchResponse{
+ Event: &flyteapp.WatchResponse_CreateEvent{
+ CreateEvent: &flyteapp.CreateEvent{App: app},
+ },
+ }); err != nil {
+ return err
+ }
+ }
+
+ for {
+ select {
+ case <-ctx.Done():
+ return nil
+ case event, ok := <-ch:
+ if !ok {
+ return nil
+ }
+ if err := stream.Send(event); err != nil {
+ return err
+ }
+ }
+ }
+}
diff --git a/app/internal/service/internal_app_service_test.go b/app/internal/service/internal_app_service_test.go
new file mode 100644
index 00000000000..c47fa16c9b8
--- /dev/null
+++ b/app/internal/service/internal_app_service_test.go
@@ -0,0 +1,373 @@
+package service
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "connectrpc.com/connect"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+ "github.com/stretchr/testify/require"
+ kerrors "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+
+ appconfig "github.com/flyteorg/flyte/v2/app/internal/config"
+ flyteapp "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app/appconnect"
+ flytecoreapp "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common"
+)
+
+// mockAppK8sClient is a testify mock for AppK8sClientInterface.
+type mockAppK8sClient struct {
+ mock.Mock
+}
+
+func (m *mockAppK8sClient) Deploy(ctx context.Context, app *flyteapp.App) error {
+ return m.Called(ctx, app).Error(0)
+}
+
+func (m *mockAppK8sClient) Stop(ctx context.Context, appID *flyteapp.Identifier) error {
+ return m.Called(ctx, appID).Error(0)
+}
+
+func (m *mockAppK8sClient) Delete(ctx context.Context, appID *flyteapp.Identifier) error {
+ return m.Called(ctx, appID).Error(0)
+}
+
+func (m *mockAppK8sClient) GetStatus(ctx context.Context, appID *flyteapp.Identifier) (*flyteapp.Status, error) {
+ args := m.Called(ctx, appID)
+ if args.Get(0) == nil {
+ return nil, args.Error(1)
+ }
+ return args.Get(0).(*flyteapp.Status), args.Error(1)
+}
+
+func (m *mockAppK8sClient) List(ctx context.Context, project, domain, appName string, limit uint32, token string) ([]*flyteapp.App, string, error) {
+ args := m.Called(ctx, project, domain, appName, limit, token)
+ if args.Get(0) == nil {
+ return nil, "", args.Error(2)
+ }
+ return args.Get(0).([]*flyteapp.App), args.String(1), args.Error(2)
+}
+
+func (m *mockAppK8sClient) GetReplicas(ctx context.Context, appID *flyteapp.Identifier) ([]*flyteapp.Replica, error) {
+ args := m.Called(ctx, appID)
+ if args.Get(0) == nil {
+ return nil, args.Error(1)
+ }
+ return args.Get(0).([]*flyteapp.Replica), args.Error(1)
+}
+
+func (m *mockAppK8sClient) DeleteReplica(ctx context.Context, replicaID *flyteapp.ReplicaIdentifier) error {
+ return m.Called(ctx, replicaID).Error(0)
+}
+
+func (m *mockAppK8sClient) Watch(ctx context.Context, project, domain, appName string) (<-chan *flyteapp.WatchResponse, error) {
+ args := m.Called(ctx, project, domain, appName)
+ if args.Get(0) == nil {
+ return nil, args.Error(1)
+ }
+ return args.Get(0).(<-chan *flyteapp.WatchResponse), args.Error(1)
+}
+
+// --- helpers ---
+
+func testCfg() *appconfig.InternalAppConfig {
+ return &appconfig.InternalAppConfig{
+ Enabled: true,
+ BaseDomain: "apps.example.com",
+ DefaultRequestTimeout: 5 * time.Minute,
+ MaxRequestTimeout: time.Hour,
+ }
+}
+
+func testAppID() *flyteapp.Identifier {
+ return &flyteapp.Identifier{Project: "proj", Domain: "dev", Name: "myapp"}
+}
+
+func testApp() *flyteapp.App {
+ return &flyteapp.App{
+ Metadata: &flyteapp.Meta{Id: testAppID()},
+ Spec: &flyteapp.Spec{
+ AppPayload: &flyteapp.Spec_Container{
+ Container: &flytecoreapp.Container{Image: "nginx:latest"},
+ },
+ },
+ }
+}
+
+func testStatus(phase flyteapp.Status_DeploymentStatus) *flyteapp.Status {
+ return &flyteapp.Status{
+ Conditions: []*flyteapp.Condition{
+ {DeploymentStatus: phase},
+ },
+ }
+}
+
+func newTestClient(t *testing.T, k8s *mockAppK8sClient) appconnect.AppServiceClient {
+ svc := NewInternalAppService(k8s, testCfg())
+ path, handler := appconnect.NewAppServiceHandler(svc)
+ mux := http.NewServeMux()
+ mux.Handle("/internal"+path, http.StripPrefix("/internal", handler))
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+ return appconnect.NewAppServiceClient(http.DefaultClient, server.URL+"/internal")
+}
+
+// --- Create ---
+
+func TestCreate_Success(t *testing.T) {
+ k8s := &mockAppK8sClient{}
+ svc := NewInternalAppService(k8s, testCfg())
+
+ app := testApp()
+ k8s.On("Deploy", mock.Anything, app).Return(nil)
+
+ resp, err := svc.Create(context.Background(), connect.NewRequest(&flyteapp.CreateRequest{App: app}))
+ require.NoError(t, err)
+ assert.Equal(t, flyteapp.Status_DEPLOYMENT_STATUS_PENDING, resp.Msg.App.Status.Conditions[0].DeploymentStatus)
+ assert.Equal(t, "https://myapp-proj-dev.apps.example.com", resp.Msg.App.Status.Ingress.PublicUrl)
+ k8s.AssertExpectations(t)
+}
+
+func TestCreate_MissingID(t *testing.T) {
+ svc := NewInternalAppService(&mockAppK8sClient{}, testCfg())
+
+ _, err := svc.Create(context.Background(), connect.NewRequest(&flyteapp.CreateRequest{
+ App: &flyteapp.App{Spec: testApp().Spec},
+ }))
+ require.Error(t, err)
+ assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err))
+}
+
+func TestCreate_MissingSpec(t *testing.T) {
+ svc := NewInternalAppService(&mockAppK8sClient{}, testCfg())
+
+ _, err := svc.Create(context.Background(), connect.NewRequest(&flyteapp.CreateRequest{
+ App: &flyteapp.App{Metadata: &flyteapp.Meta{Id: testAppID()}},
+ }))
+ require.Error(t, err)
+ assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err))
+}
+
+func TestCreate_MissingPayload(t *testing.T) {
+ svc := NewInternalAppService(&mockAppK8sClient{}, testCfg())
+
+ _, err := svc.Create(context.Background(), connect.NewRequest(&flyteapp.CreateRequest{
+ App: &flyteapp.App{
+ Metadata: &flyteapp.Meta{Id: testAppID()},
+ Spec: &flyteapp.Spec{},
+ },
+ }))
+ require.Error(t, err)
+ assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err))
+}
+
+func TestCreate_NoBaseDomain_NoIngress(t *testing.T) {
+ k8s := &mockAppK8sClient{}
+ cfg := testCfg()
+ cfg.BaseDomain = ""
+ svc := NewInternalAppService(k8s, cfg)
+
+ app := testApp()
+ k8s.On("Deploy", mock.Anything, app).Return(nil)
+
+ resp, err := svc.Create(context.Background(), connect.NewRequest(&flyteapp.CreateRequest{App: app}))
+ require.NoError(t, err)
+ assert.Nil(t, resp.Msg.App.Status.Ingress)
+ k8s.AssertExpectations(t)
+}
+
+// --- Get ---
+
+func TestGet_Success(t *testing.T) {
+ k8s := &mockAppK8sClient{}
+ svc := NewInternalAppService(k8s, testCfg())
+
+ appID := testAppID()
+ k8s.On("GetStatus", mock.Anything, appID).Return(testStatus(flyteapp.Status_DEPLOYMENT_STATUS_ACTIVE), nil)
+
+ resp, err := svc.Get(context.Background(), connect.NewRequest(&flyteapp.GetRequest{
+ Identifier: &flyteapp.GetRequest_AppId{AppId: appID},
+ }))
+ require.NoError(t, err)
+ assert.Equal(t, flyteapp.Status_DEPLOYMENT_STATUS_ACTIVE, resp.Msg.App.Status.Conditions[0].DeploymentStatus)
+ k8s.AssertExpectations(t)
+}
+
+func TestGet_NotFound(t *testing.T) {
+ k8s := &mockAppK8sClient{}
+ svc := NewInternalAppService(k8s, testCfg())
+
+ appID := testAppID()
+ notFoundErr := fmt.Errorf("KService myapp not found: %w", kerrors.NewNotFound(schema.GroupResource{}, "myapp"))
+ k8s.On("GetStatus", mock.Anything, appID).Return(nil, notFoundErr)
+
+ _, err := svc.Get(context.Background(), connect.NewRequest(&flyteapp.GetRequest{
+ Identifier: &flyteapp.GetRequest_AppId{AppId: appID},
+ }))
+ require.Error(t, err)
+ assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err))
+ k8s.AssertExpectations(t)
+}
+
+func TestGet_MissingAppID(t *testing.T) {
+ svc := NewInternalAppService(&mockAppK8sClient{}, testCfg())
+
+ _, err := svc.Get(context.Background(), connect.NewRequest(&flyteapp.GetRequest{}))
+ require.Error(t, err)
+ assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err))
+}
+
+// --- Update ---
+
+func TestUpdate_Deploy(t *testing.T) {
+ k8s := &mockAppK8sClient{}
+ svc := NewInternalAppService(k8s, testCfg())
+
+ app := testApp()
+ k8s.On("Deploy", mock.Anything, app).Return(nil)
+ k8s.On("GetStatus", mock.Anything, app.Metadata.Id).Return(testStatus(flyteapp.Status_DEPLOYMENT_STATUS_DEPLOYING), nil)
+
+ resp, err := svc.Update(context.Background(), connect.NewRequest(&flyteapp.UpdateRequest{App: app}))
+ require.NoError(t, err)
+ assert.Equal(t, flyteapp.Status_DEPLOYMENT_STATUS_DEPLOYING, resp.Msg.App.Status.Conditions[0].DeploymentStatus)
+ k8s.AssertExpectations(t)
+}
+
+func TestUpdate_Stop(t *testing.T) {
+ k8s := &mockAppK8sClient{}
+ svc := NewInternalAppService(k8s, testCfg())
+
+ app := testApp()
+ app.Spec.DesiredState = flyteapp.Spec_DESIRED_STATE_STOPPED
+ k8s.On("Stop", mock.Anything, app.Metadata.Id).Return(nil)
+ k8s.On("GetStatus", mock.Anything, app.Metadata.Id).Return(testStatus(flyteapp.Status_DEPLOYMENT_STATUS_STOPPED), nil)
+
+ resp, err := svc.Update(context.Background(), connect.NewRequest(&flyteapp.UpdateRequest{App: app}))
+ require.NoError(t, err)
+ assert.Equal(t, flyteapp.Status_DEPLOYMENT_STATUS_STOPPED, resp.Msg.App.Status.Conditions[0].DeploymentStatus)
+ k8s.AssertExpectations(t)
+}
+
+func TestUpdate_MissingID(t *testing.T) {
+ svc := NewInternalAppService(&mockAppK8sClient{}, testCfg())
+
+ _, err := svc.Update(context.Background(), connect.NewRequest(&flyteapp.UpdateRequest{
+ App: &flyteapp.App{},
+ }))
+ require.Error(t, err)
+ assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err))
+}
+
+// --- Delete ---
+
+func TestDelete_Success(t *testing.T) {
+ k8s := &mockAppK8sClient{}
+ svc := NewInternalAppService(k8s, testCfg())
+
+ appID := testAppID()
+ k8s.On("Delete", mock.Anything, appID).Return(nil)
+
+ _, err := svc.Delete(context.Background(), connect.NewRequest(&flyteapp.DeleteRequest{AppId: appID}))
+ require.NoError(t, err)
+ k8s.AssertExpectations(t)
+}
+
+func TestDelete_MissingID(t *testing.T) {
+ svc := NewInternalAppService(&mockAppK8sClient{}, testCfg())
+
+ _, err := svc.Delete(context.Background(), connect.NewRequest(&flyteapp.DeleteRequest{}))
+ require.Error(t, err)
+ assert.Equal(t, connect.CodeInvalidArgument, connect.CodeOf(err))
+}
+
+// --- List ---
+
+func TestList_ByProject(t *testing.T) {
+ k8s := &mockAppK8sClient{}
+ svc := NewInternalAppService(k8s, testCfg())
+
+ apps := []*flyteapp.App{testApp()}
+ k8s.On("List", mock.Anything, "proj", "dev", "", uint32(10), "tok").Return(apps, "nexttok", nil)
+
+ resp, err := svc.List(context.Background(), connect.NewRequest(&flyteapp.ListRequest{
+ FilterBy: &flyteapp.ListRequest_Project{
+ Project: &common.ProjectIdentifier{Name: "proj", Domain: "dev"},
+ },
+ Request: &common.ListRequest{Limit: 10, Token: "tok"},
+ }))
+ require.NoError(t, err)
+ assert.Len(t, resp.Msg.Apps, 1)
+ assert.Equal(t, "nexttok", resp.Msg.Token)
+ k8s.AssertExpectations(t)
+}
+
+func TestList_NoFilter(t *testing.T) {
+ k8s := &mockAppK8sClient{}
+ svc := NewInternalAppService(k8s, testCfg())
+
+ k8s.On("List", mock.Anything, "", "", "", uint32(0), "").Return([]*flyteapp.App{}, "", nil)
+
+ resp, err := svc.List(context.Background(), connect.NewRequest(&flyteapp.ListRequest{}))
+ require.NoError(t, err)
+ assert.Empty(t, resp.Msg.Apps)
+ k8s.AssertExpectations(t)
+}
+
+// --- Watch ---
+
+func TestWatch_InitialSnapshot(t *testing.T) {
+ k8s := &mockAppK8sClient{}
+
+ apps := []*flyteapp.App{testApp()}
+ ch := make(chan *flyteapp.WatchResponse)
+ close(ch)
+
+ k8s.On("Watch", mock.Anything, "proj", "dev", "").Return((<-chan *flyteapp.WatchResponse)(ch), nil)
+ k8s.On("List", mock.Anything, "proj", "dev", "", uint32(0), "").Return(apps, "", nil)
+
+ client := newTestClient(t, k8s)
+ stream, err := client.Watch(context.Background(), connect.NewRequest(&flyteapp.WatchRequest{
+ Target: &flyteapp.WatchRequest_Project{
+ Project: &common.ProjectIdentifier{Name: "proj", Domain: "dev"},
+ },
+ }))
+ require.NoError(t, err)
+
+ // Expect one CreateEvent from the initial snapshot.
+ require.True(t, stream.Receive())
+ resp := stream.Msg()
+ ce, ok := resp.Event.(*flyteapp.WatchResponse_CreateEvent)
+ require.True(t, ok)
+ assert.Equal(t, "myapp", ce.CreateEvent.App.Metadata.Id.Name)
+
+ // Channel is closed — stream should end.
+ assert.False(t, stream.Receive())
+ k8s.AssertExpectations(t)
+}
+
+func TestWatch_AppIDTarget(t *testing.T) {
+ k8s := &mockAppK8sClient{}
+
+ ch := make(chan *flyteapp.WatchResponse)
+ close(ch)
+
+ k8s.On("Watch", mock.Anything, "proj", "dev", "myapp").Return((<-chan *flyteapp.WatchResponse)(ch), nil)
+ k8s.On("List", mock.Anything, "proj", "dev", "myapp", uint32(0), "").Return([]*flyteapp.App{}, "", nil)
+
+ client := newTestClient(t, k8s)
+ stream, err := client.Watch(context.Background(), connect.NewRequest(&flyteapp.WatchRequest{
+ Target: &flyteapp.WatchRequest_AppId{AppId: testAppID()},
+ }))
+ require.NoError(t, err)
+
+ // No snapshot apps, channel closed — stream ends immediately.
+ assert.False(t, stream.Receive())
+ k8s.AssertExpectations(t)
+}
diff --git a/app/internal/setup.go b/app/internal/setup.go
new file mode 100644
index 00000000000..7c7ec7a0c53
--- /dev/null
+++ b/app/internal/setup.go
@@ -0,0 +1,38 @@
+package internal
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+
+ stdlibapp "github.com/flyteorg/flyte/v2/flytestdlib/app"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+
+ appconfig "github.com/flyteorg/flyte/v2/app/internal/config"
+ appk8s "github.com/flyteorg/flyte/v2/app/internal/k8s"
+ "github.com/flyteorg/flyte/v2/app/internal/service"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app/appconnect"
+)
+
+// Setup registers the InternalAppService handler on the SetupContext mux.
+// It is mounted at /internal to avoid collision with the control plane
+// AppService, which shares the same proto service definition.
+func Setup(ctx context.Context, sc *stdlibapp.SetupContext, cfg *appconfig.InternalAppConfig) error {
+ if !cfg.Enabled {
+ logger.Infof(ctx, "InternalAppService disabled (apps.enabled=false), skipping setup")
+ return nil
+ }
+
+ if err := stdlibapp.InitAppScheme(); err != nil {
+ return fmt.Errorf("internalapp: failed to register Knative scheme: %w", err)
+ }
+
+ appK8sClient := appk8s.NewAppK8sClient(sc.K8sClient, sc.K8sCache, cfg)
+ internalAppSvc := service.NewInternalAppService(appK8sClient, cfg)
+
+ path, handler := appconnect.NewAppServiceHandler(internalAppSvc)
+ sc.Mux.Handle("/internal"+path, http.StripPrefix("/internal", handler))
+ logger.Infof(ctx, "Mounted InternalAppService at /internal%s", path)
+
+ return nil
+}
diff --git a/app/service/app_service.go b/app/service/app_service.go
new file mode 100644
index 00000000000..55b70738117
--- /dev/null
+++ b/app/service/app_service.go
@@ -0,0 +1,141 @@
+package service
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "connectrpc.com/connect"
+ "github.com/hashicorp/golang-lru/v2/expirable"
+
+ flyteapp "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app/appconnect"
+)
+
+// AppService is the control plane implementation of AppServiceHandler.
+// It proxies all RPCs to InternalAppService (data plane) and maintains a
+// per-instance TTL cache to reduce cross-plane RPC calls on Get.
+type AppService struct {
+ appconnect.UnimplementedAppServiceHandler
+ internalClient appconnect.AppServiceClient
+ // cache is nil when cacheTTL=0 (caching disabled).
+ cache *expirable.LRU[string, *flyteapp.App]
+}
+
+// NewAppService creates a new AppService.
+// cacheTTL=0 disables caching (every Get calls InternalAppService).
+func NewAppService(internalClient appconnect.AppServiceClient, cacheTTL time.Duration) *AppService {
+ var cache *expirable.LRU[string, *flyteapp.App]
+ if cacheTTL > 0 {
+ cache = expirable.NewLRU[string, *flyteapp.App](0, nil, cacheTTL)
+ }
+ return &AppService{
+ internalClient: internalClient,
+ cache: cache,
+ }
+}
+
+// Ensure AppService satisfies the generated handler interface.
+var _ appconnect.AppServiceHandler = (*AppService)(nil)
+
+// Create forwards to InternalAppService and invalidates the cache entry.
+func (s *AppService) Create(
+ ctx context.Context,
+ req *connect.Request[flyteapp.CreateRequest],
+) (*connect.Response[flyteapp.CreateResponse], error) {
+ resp, err := s.internalClient.Create(ctx, req)
+ if err != nil {
+ return nil, err
+ }
+ if s.cache != nil {
+ s.cache.Remove(cacheKey(req.Msg.GetApp().GetMetadata().GetId()))
+ }
+ return resp, nil
+}
+
+// Get returns the app, using the cache on hit and calling InternalAppService on miss.
+func (s *AppService) Get(
+ ctx context.Context,
+ req *connect.Request[flyteapp.GetRequest],
+) (*connect.Response[flyteapp.GetResponse], error) {
+ appID, ok := req.Msg.GetIdentifier().(*flyteapp.GetRequest_AppId)
+ if ok && appID.AppId != nil && s.cache != nil {
+ if app, hit := s.cache.Get(cacheKey(appID.AppId)); hit {
+ return connect.NewResponse(&flyteapp.GetResponse{App: app}), nil
+ }
+ }
+
+ resp, err := s.internalClient.Get(ctx, req)
+ if err != nil {
+ return nil, err
+ }
+ if ok && appID.AppId != nil && s.cache != nil {
+ s.cache.Add(cacheKey(appID.AppId), resp.Msg.GetApp())
+ }
+ return resp, nil
+}
+
+// Update forwards to InternalAppService and invalidates the cache entry.
+func (s *AppService) Update(
+ ctx context.Context,
+ req *connect.Request[flyteapp.UpdateRequest],
+) (*connect.Response[flyteapp.UpdateResponse], error) {
+ resp, err := s.internalClient.Update(ctx, req)
+ if err != nil {
+ return nil, err
+ }
+ if s.cache != nil {
+ s.cache.Remove(cacheKey(req.Msg.GetApp().GetMetadata().GetId()))
+ }
+ return resp, nil
+}
+
+// Delete forwards to InternalAppService and invalidates the cache entry.
+func (s *AppService) Delete(
+ ctx context.Context,
+ req *connect.Request[flyteapp.DeleteRequest],
+) (*connect.Response[flyteapp.DeleteResponse], error) {
+ resp, err := s.internalClient.Delete(ctx, req)
+ if err != nil {
+ return nil, err
+ }
+ if s.cache != nil {
+ s.cache.Remove(cacheKey(req.Msg.GetAppId()))
+ }
+ return resp, nil
+}
+
+// List always forwards to InternalAppService — results vary by filter/pagination.
+func (s *AppService) List(
+ ctx context.Context,
+ req *connect.Request[flyteapp.ListRequest],
+) (*connect.Response[flyteapp.ListResponse], error) {
+ return s.internalClient.List(ctx, req)
+}
+
+// Watch proxies the server-streaming Watch RPC to InternalAppService.
+func (s *AppService) Watch(
+ ctx context.Context,
+ req *connect.Request[flyteapp.WatchRequest],
+ stream *connect.ServerStream[flyteapp.WatchResponse],
+) error {
+ clientStream, err := s.internalClient.Watch(ctx, req)
+ if err != nil {
+ return connect.NewError(connect.CodeInternal, err)
+ }
+ defer clientStream.Close()
+ for clientStream.Receive() {
+ if err := stream.Send(clientStream.Msg()); err != nil {
+ return err
+ }
+ }
+ return clientStream.Err()
+}
+
+// cacheKey returns a stable string key for an app identifier.
+func cacheKey(id *flyteapp.Identifier) string {
+ if id == nil {
+ return ""
+ }
+ return fmt.Sprintf("%s/%s/%s", id.GetProject(), id.GetDomain(), id.GetName())
+}
diff --git a/app/service/app_service_test.go b/app/service/app_service_test.go
new file mode 100644
index 00000000000..7b623b8ff8b
--- /dev/null
+++ b/app/service/app_service_test.go
@@ -0,0 +1,288 @@
+package service
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "connectrpc.com/connect"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+ "github.com/stretchr/testify/require"
+
+ flyteapp "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app/appconnect"
+ flytecoreapp "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+// mockInternalClient is a testify mock for appconnect.AppServiceClient.
+type mockInternalClient struct {
+ mock.Mock
+}
+
+func (m *mockInternalClient) Create(ctx context.Context, req *connect.Request[flyteapp.CreateRequest]) (*connect.Response[flyteapp.CreateResponse], error) {
+ args := m.Called(ctx, req)
+ if args.Get(0) == nil {
+ return nil, args.Error(1)
+ }
+ return args.Get(0).(*connect.Response[flyteapp.CreateResponse]), args.Error(1)
+}
+
+func (m *mockInternalClient) Get(ctx context.Context, req *connect.Request[flyteapp.GetRequest]) (*connect.Response[flyteapp.GetResponse], error) {
+ args := m.Called(ctx, req)
+ if args.Get(0) == nil {
+ return nil, args.Error(1)
+ }
+ return args.Get(0).(*connect.Response[flyteapp.GetResponse]), args.Error(1)
+}
+
+func (m *mockInternalClient) Update(ctx context.Context, req *connect.Request[flyteapp.UpdateRequest]) (*connect.Response[flyteapp.UpdateResponse], error) {
+ args := m.Called(ctx, req)
+ if args.Get(0) == nil {
+ return nil, args.Error(1)
+ }
+ return args.Get(0).(*connect.Response[flyteapp.UpdateResponse]), args.Error(1)
+}
+
+func (m *mockInternalClient) UpdateStatus(ctx context.Context, req *connect.Request[flyteapp.UpdateStatusRequest]) (*connect.Response[flyteapp.UpdateStatusResponse], error) {
+ return nil, connect.NewError(connect.CodeUnimplemented, nil)
+}
+
+func (m *mockInternalClient) Delete(ctx context.Context, req *connect.Request[flyteapp.DeleteRequest]) (*connect.Response[flyteapp.DeleteResponse], error) {
+ args := m.Called(ctx, req)
+ if args.Get(0) == nil {
+ return nil, args.Error(1)
+ }
+ return args.Get(0).(*connect.Response[flyteapp.DeleteResponse]), args.Error(1)
+}
+
+func (m *mockInternalClient) List(ctx context.Context, req *connect.Request[flyteapp.ListRequest]) (*connect.Response[flyteapp.ListResponse], error) {
+ args := m.Called(ctx, req)
+ if args.Get(0) == nil {
+ return nil, args.Error(1)
+ }
+ return args.Get(0).(*connect.Response[flyteapp.ListResponse]), args.Error(1)
+}
+
+func (m *mockInternalClient) Watch(ctx context.Context, req *connect.Request[flyteapp.WatchRequest]) (*connect.ServerStreamForClient[flyteapp.WatchResponse], error) {
+ args := m.Called(ctx, req)
+ if args.Get(0) == nil {
+ return nil, args.Error(1)
+ }
+ return args.Get(0).(*connect.ServerStreamForClient[flyteapp.WatchResponse]), args.Error(1)
+}
+
+func (m *mockInternalClient) Lease(ctx context.Context, req *connect.Request[flyteapp.LeaseRequest]) (*connect.ServerStreamForClient[flyteapp.LeaseResponse], error) {
+ args := m.Called(ctx, req)
+ if args.Get(0) == nil {
+ return nil, args.Error(1)
+ }
+ return args.Get(0).(*connect.ServerStreamForClient[flyteapp.LeaseResponse]), args.Error(1)
+}
+
+// --- helpers ---
+
+func testAppID() *flyteapp.Identifier {
+ return &flyteapp.Identifier{Project: "proj", Domain: "dev", Name: "myapp"}
+}
+
+func testApp() *flyteapp.App {
+ return &flyteapp.App{
+ Metadata: &flyteapp.Meta{Id: testAppID()},
+ Spec: &flyteapp.Spec{
+ AppPayload: &flyteapp.Spec_Container{
+ Container: &flytecoreapp.Container{Image: "nginx:latest"},
+ },
+ },
+ Status: &flyteapp.Status{
+ Conditions: []*flyteapp.Condition{
+ {DeploymentStatus: flyteapp.Status_DEPLOYMENT_STATUS_ACTIVE},
+ },
+ },
+ }
+}
+
+// --- Get with cache ---
+
+func TestGet_CacheMiss_CallsInternal(t *testing.T) {
+ internal := &mockInternalClient{}
+ svc := NewAppService(internal, 30*time.Second)
+
+ appID := testAppID()
+ app := testApp()
+ internal.On("Get", mock.Anything, mock.Anything).Return(
+ connect.NewResponse(&flyteapp.GetResponse{App: app}), nil,
+ )
+
+ resp, err := svc.Get(context.Background(), connect.NewRequest(&flyteapp.GetRequest{
+ Identifier: &flyteapp.GetRequest_AppId{AppId: appID},
+ }))
+ require.NoError(t, err)
+ assert.Equal(t, flyteapp.Status_DEPLOYMENT_STATUS_ACTIVE, resp.Msg.App.Status.Conditions[0].DeploymentStatus)
+ internal.AssertExpectations(t)
+}
+
+func TestGet_CacheHit_SkipsInternal(t *testing.T) {
+ internal := &mockInternalClient{}
+ svc := NewAppService(internal, 30*time.Second)
+
+ // Pre-populate cache.
+ appID := testAppID()
+ svc.cache.Add(cacheKey(appID), testApp())
+
+ // Internal should NOT be called.
+ resp, err := svc.Get(context.Background(), connect.NewRequest(&flyteapp.GetRequest{
+ Identifier: &flyteapp.GetRequest_AppId{AppId: appID},
+ }))
+ require.NoError(t, err)
+ assert.Equal(t, flyteapp.Status_DEPLOYMENT_STATUS_ACTIVE, resp.Msg.App.Status.Conditions[0].DeploymentStatus)
+ internal.AssertNotCalled(t, "Get")
+}
+
+func TestGet_CacheExpired_CallsInternal(t *testing.T) {
+ internal := &mockInternalClient{}
+ svc := NewAppService(internal, 1*time.Millisecond)
+
+ appID := testAppID()
+ svc.cache.Add(cacheKey(appID), testApp())
+ time.Sleep(5 * time.Millisecond) // let TTL expire
+
+ app := testApp()
+ internal.On("Get", mock.Anything, mock.Anything).Return(
+ connect.NewResponse(&flyteapp.GetResponse{App: app}), nil,
+ )
+
+ _, err := svc.Get(context.Background(), connect.NewRequest(&flyteapp.GetRequest{
+ Identifier: &flyteapp.GetRequest_AppId{AppId: appID},
+ }))
+ require.NoError(t, err)
+ internal.AssertExpectations(t)
+}
+
+// --- Create / Update / Delete invalidate cache ---
+
+func TestCreate_InvalidatesCache(t *testing.T) {
+ internal := &mockInternalClient{}
+ svc := NewAppService(internal, 30*time.Second)
+
+ app := testApp()
+ // Pre-populate cache so we can confirm it's cleared.
+ svc.cache.Add(cacheKey(app.Metadata.Id), app)
+
+ internal.On("Create", mock.Anything, mock.Anything).Return(
+ connect.NewResponse(&flyteapp.CreateResponse{App: app}), nil,
+ )
+
+ _, err := svc.Create(context.Background(), connect.NewRequest(&flyteapp.CreateRequest{App: app}))
+ require.NoError(t, err)
+
+ _, hit := svc.cache.Get(cacheKey(app.Metadata.Id))
+ assert.False(t, hit, "cache should be invalidated after Create")
+ internal.AssertExpectations(t)
+}
+
+func TestUpdate_InvalidatesCache(t *testing.T) {
+ internal := &mockInternalClient{}
+ svc := NewAppService(internal, 30*time.Second)
+
+ app := testApp()
+ svc.cache.Add(cacheKey(app.Metadata.Id), app)
+
+ internal.On("Update", mock.Anything, mock.Anything).Return(
+ connect.NewResponse(&flyteapp.UpdateResponse{App: app}), nil,
+ )
+
+ _, err := svc.Update(context.Background(), connect.NewRequest(&flyteapp.UpdateRequest{App: app}))
+ require.NoError(t, err)
+
+ _, hit := svc.cache.Get(cacheKey(app.Metadata.Id))
+ assert.False(t, hit, "cache should be invalidated after Update")
+ internal.AssertExpectations(t)
+}
+
+func TestDelete_InvalidatesCache(t *testing.T) {
+ internal := &mockInternalClient{}
+ svc := NewAppService(internal, 30*time.Second)
+
+ appID := testAppID()
+ svc.cache.Add(cacheKey(appID), testApp())
+
+ internal.On("Delete", mock.Anything, mock.Anything).Return(
+ connect.NewResponse(&flyteapp.DeleteResponse{}), nil,
+ )
+
+ _, err := svc.Delete(context.Background(), connect.NewRequest(&flyteapp.DeleteRequest{AppId: appID}))
+ require.NoError(t, err)
+
+ _, hit := svc.cache.Get(cacheKey(appID))
+ assert.False(t, hit, "cache should be invalidated after Delete")
+ internal.AssertExpectations(t)
+}
+
+// --- List always forwards ---
+
+func TestList_AlwaysCallsInternal(t *testing.T) {
+ internal := &mockInternalClient{}
+ svc := NewAppService(internal, 30*time.Second)
+
+ internal.On("List", mock.Anything, mock.Anything).Return(
+ connect.NewResponse(&flyteapp.ListResponse{Apps: []*flyteapp.App{testApp()}}), nil,
+ )
+
+ resp, err := svc.List(context.Background(), connect.NewRequest(&flyteapp.ListRequest{}))
+ require.NoError(t, err)
+ assert.Len(t, resp.Msg.Apps, 1)
+ internal.AssertExpectations(t)
+}
+
+// --- Watch streams through ---
+
+func TestWatch_ProxiesStream(t *testing.T) {
+ // Use a real httptest server to exercise the streaming path.
+ internal := &mockInternalClient{}
+ svc := NewAppService(internal, 30*time.Second)
+
+ path, handler := appconnect.NewAppServiceHandler(svc)
+ mux := http.NewServeMux()
+ mux.Handle(path, handler)
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+
+ // Mount InternalAppService on the same test server at /internal so the
+ // proxy can route to it.
+ internalSvcPath, internalSvcHandler := appconnect.NewAppServiceHandler(
+ &echoWatchService{app: testApp()},
+ )
+ mux.Handle("/internal"+internalSvcPath, http.StripPrefix("/internal", internalSvcHandler))
+
+ // Point AppService proxy at the internal path on the same server.
+ svc.internalClient = appconnect.NewAppServiceClient(http.DefaultClient, server.URL+"/internal")
+
+ client := appconnect.NewAppServiceClient(http.DefaultClient, server.URL)
+ stream, err := client.Watch(context.Background(), connect.NewRequest(&flyteapp.WatchRequest{}))
+ require.NoError(t, err)
+
+ require.True(t, stream.Receive())
+ assert.Equal(t, "myapp", stream.Msg().GetCreateEvent().GetApp().GetMetadata().GetId().GetName())
+ stream.Close()
+}
+
+// echoWatchService sends one CreateEvent then closes the stream.
+type echoWatchService struct {
+ appconnect.UnimplementedAppServiceHandler
+ app *flyteapp.App
+}
+
+func (e *echoWatchService) Watch(
+ _ context.Context,
+ _ *connect.Request[flyteapp.WatchRequest],
+ stream *connect.ServerStream[flyteapp.WatchResponse],
+) error {
+ return stream.Send(&flyteapp.WatchResponse{
+ Event: &flyteapp.WatchResponse_CreateEvent{
+ CreateEvent: &flyteapp.CreateEvent{App: e.app},
+ },
+ })
+}
diff --git a/app/setup.go b/app/setup.go
new file mode 100644
index 00000000000..31fcdd8d6a8
--- /dev/null
+++ b/app/setup.go
@@ -0,0 +1,36 @@
+package app
+
+import (
+ "context"
+ "net/http"
+
+ stdlibapp "github.com/flyteorg/flyte/v2/flytestdlib/app"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+
+ appconfig "github.com/flyteorg/flyte/v2/app/config"
+ "github.com/flyteorg/flyte/v2/app/service"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app/appconnect"
+)
+
+// Setup registers the control plane AppService handler on the SetupContext mux.
+// In unified mode (sc.BaseURL set), the proxy routes to InternalAppService on
+// the same mux via the /internal prefix — no network hop. In split mode,
+// cfg.InternalAppServiceURL points at the data plane host.
+func Setup(ctx context.Context, sc *stdlibapp.SetupContext, cfg *appconfig.AppConfig) error {
+ internalAppURL := cfg.InternalAppServiceURL
+ if sc.BaseURL != "" {
+ internalAppURL = sc.BaseURL
+ }
+
+ internalClient := appconnect.NewAppServiceClient(
+ http.DefaultClient,
+ internalAppURL+"/internal",
+ )
+
+ appSvc := service.NewAppService(internalClient, cfg.CacheTTL)
+ path, handler := appconnect.NewAppServiceHandler(appSvc)
+ sc.Mux.Handle(path, handler)
+ logger.Infof(ctx, "Mounted AppService at %s", path)
+
+ return nil
+}
diff --git a/boilerplate/flyte/code_of_conduct/CODE_OF_CONDUCT.md b/boilerplate/flyte/code_of_conduct/CODE_OF_CONDUCT.md
new file mode 100644
index 00000000000..e12139d6916
--- /dev/null
+++ b/boilerplate/flyte/code_of_conduct/CODE_OF_CONDUCT.md
@@ -0,0 +1,2 @@
+This project is governed by LF AI Foundation's [code of conduct](https://lfprojects.org/policies/code-of-conduct/).
+All contributors and participants agree to abide by its terms.
diff --git a/boilerplate/flyte/code_of_conduct/README.rst b/boilerplate/flyte/code_of_conduct/README.rst
new file mode 100644
index 00000000000..0c9f2f1ec52
--- /dev/null
+++ b/boilerplate/flyte/code_of_conduct/README.rst
@@ -0,0 +1,2 @@
+CODE OF CONDUCT
+~~~~~~~~~~~~~~~
diff --git a/boilerplate/flyte/code_of_conduct/update.sh b/boilerplate/flyte/code_of_conduct/update.sh
new file mode 100755
index 00000000000..42f61584603
--- /dev/null
+++ b/boilerplate/flyte/code_of_conduct/update.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+cp ${DIR}/CODE_OF_CONDUCT.md ${DIR}/../../../CODE_OF_CONDUCT.md
diff --git a/boilerplate/flyte/docker_build/Makefile b/boilerplate/flyte/docker_build/Makefile
new file mode 100644
index 00000000000..e2b2b8a18dc
--- /dev/null
+++ b/boilerplate/flyte/docker_build/Makefile
@@ -0,0 +1,12 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+.PHONY: docker_build
+docker_build:
+ IMAGE_NAME=$$REPOSITORY ./boilerplate/flyte/docker_build/docker_build.sh
+
+.PHONY: dockerhub_push
+dockerhub_push:
+ IMAGE_NAME=flyteorg/$$REPOSITORY REGISTRY=docker.io ./boilerplate/flyte/docker_build/docker_build.sh
diff --git a/boilerplate/flyte/docker_build/Readme.rst b/boilerplate/flyte/docker_build/Readme.rst
new file mode 100644
index 00000000000..7790b8fbfd2
--- /dev/null
+++ b/boilerplate/flyte/docker_build/Readme.rst
@@ -0,0 +1,23 @@
+Docker Build and Push
+~~~~~~~~~~~~~~~~~~~~~
+
+Provides a ``make docker_build`` target that builds your image locally.
+
+Provides a ``make dockerhub_push`` target that pushes your final image to Dockerhub.
+
+The Dockerhub image will tagged ``:``
+
+If git head has a git tag, the Dockerhub image will also be tagged ``:``.
+
+**To Enable:**
+
+Add ``flyteorg/docker_build`` to your ``boilerplate/update.cfg`` file.
+
+Add ``include boilerplate/flyte/docker_build/Makefile`` in your main ``Makefile`` _after_ your REPOSITORY environment variable
+
+::
+
+ REPOSITORY=
+ include boilerplate/flyte/docker_build/Makefile
+
+(this ensures the extra Make targets get included in your main Makefile)
diff --git a/boilerplate/flyte/docker_build/docker_build.sh b/boilerplate/flyte/docker_build/docker_build.sh
new file mode 100755
index 00000000000..817189aee17
--- /dev/null
+++ b/boilerplate/flyte/docker_build/docker_build.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+set -e
+
+echo ""
+echo "------------------------------------"
+echo " DOCKER BUILD"
+echo "------------------------------------"
+echo ""
+
+if [ -n "$REGISTRY" ]; then
+ # Do not push if there are unstaged git changes
+ CHANGED=$(git status --porcelain)
+ if [ -n "$CHANGED" ]; then
+ echo "Please commit git changes before pushing to a registry"
+ exit 1
+ fi
+fi
+
+
+GIT_SHA=$(git rev-parse HEAD)
+
+IMAGE_TAG_SUFFIX=""
+# for intermediate build phases, append -$BUILD_PHASE to all image tags
+if [ -n "$BUILD_PHASE" ]; then
+ IMAGE_TAG_SUFFIX="-${BUILD_PHASE}"
+fi
+
+IMAGE_TAG_WITH_SHA="${IMAGE_NAME}:${GIT_SHA}${IMAGE_TAG_SUFFIX}"
+
+RELEASE_SEMVER=$(git describe --tags --exact-match "$GIT_SHA" 2>/dev/null) || true
+if [ -n "$RELEASE_SEMVER" ]; then
+ IMAGE_TAG_WITH_SEMVER="${IMAGE_NAME}:${RELEASE_SEMVER}${IMAGE_TAG_SUFFIX}"
+fi
+
+# build the image
+# passing no build phase will build the final image
+docker build -t "$IMAGE_TAG_WITH_SHA" --target=${BUILD_PHASE} .
+echo "${IMAGE_TAG_WITH_SHA} built locally."
+
+# if REGISTRY specified, push the images to the remote registry
+if [ -n "$REGISTRY" ]; then
+
+ if [ -n "${DOCKER_REGISTRY_PASSWORD}" ]; then
+ docker login --username="$DOCKER_REGISTRY_USERNAME" --password="$DOCKER_REGISTRY_PASSWORD"
+ fi
+
+ docker tag "$IMAGE_TAG_WITH_SHA" "${REGISTRY}/${IMAGE_TAG_WITH_SHA}"
+
+ docker push "${REGISTRY}/${IMAGE_TAG_WITH_SHA}"
+ echo "${REGISTRY}/${IMAGE_TAG_WITH_SHA} pushed to remote."
+
+ # If the current commit has a semver tag, also push the images with the semver tag
+ if [ -n "$RELEASE_SEMVER" ]; then
+
+ docker tag "$IMAGE_TAG_WITH_SHA" "${REGISTRY}/${IMAGE_TAG_WITH_SEMVER}"
+
+ docker push "${REGISTRY}/${IMAGE_TAG_WITH_SEMVER}"
+ echo "${REGISTRY}/${IMAGE_TAG_WITH_SEMVER} pushed to remote."
+
+ fi
+fi
diff --git a/boilerplate/flyte/end2end/Makefile b/boilerplate/flyte/end2end/Makefile
new file mode 100644
index 00000000000..983b6e22d98
--- /dev/null
+++ b/boilerplate/flyte/end2end/Makefile
@@ -0,0 +1,18 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+.PHONY: end2end_execute
+end2end_execute: export FLYTESNACKS_PRIORITIES ?= P0
+end2end_execute: export FLYTESNACKS_VERSION ?= $(shell curl --silent "https://api.github.com/repos/flyteorg/flytesnacks/releases/latest" | jq -r .tag_name)
+end2end_execute:
+ pytest ./boilerplate/flyte/end2end/test_run.py \
+ --flytesnacks_release_tag=$(FLYTESNACKS_VERSION) \
+ --priorities=$(FLYTESNACKS_PRIORITIES) \
+ --config_file=./boilerplate/flyte/end2end/functional-test-config.yaml \
+ --return_non_zero_on_failure
+
+.PHONY: k8s_integration_execute
+k8s_integration_execute:
+ echo "pass"
diff --git a/boilerplate/flyte/end2end/conftest.py b/boilerplate/flyte/end2end/conftest.py
new file mode 100644
index 00000000000..d77fad05d99
--- /dev/null
+++ b/boilerplate/flyte/end2end/conftest.py
@@ -0,0 +1,47 @@
+import pytest
+
+def pytest_addoption(parser):
+ parser.addoption("--flytesnacks_release_tag", required=True)
+ parser.addoption("--priorities", required=True)
+ parser.addoption("--config_file", required=True)
+ parser.addoption(
+ "--return_non_zero_on_failure",
+ action="store_true",
+ default=False,
+ help="Return a non-zero exit status if any workflow fails",
+ )
+ parser.addoption(
+ "--terminate_workflow_on_failure",
+ action="store_true",
+ default=False,
+ help="Abort failing workflows upon exit",
+ )
+ parser.addoption(
+ "--test_project_name",
+ default="flytesnacks",
+ help="Name of project to run functional tests on"
+ )
+ parser.addoption(
+ "--test_project_domain",
+ default="development",
+ help="Name of domain in project to run functional tests on"
+ )
+ parser.addoption(
+ "--cluster_pool_name",
+ required=False,
+ type=str,
+ default=None,
+ )
+
+@pytest.fixture
+def setup_flytesnacks_env(pytestconfig):
+ return {
+ "flytesnacks_release_tag": pytestconfig.getoption("--flytesnacks_release_tag"),
+ "priorities": pytestconfig.getoption("--priorities"),
+ "config_file": pytestconfig.getoption("--config_file"),
+ "return_non_zero_on_failure": pytestconfig.getoption("--return_non_zero_on_failure"),
+ "terminate_workflow_on_failure": pytestconfig.getoption("--terminate_workflow_on_failure"),
+ "test_project_name": pytestconfig.getoption("--test_project_name"),
+ "test_project_domain": pytestconfig.getoption("--test_project_domain"),
+ "cluster_pool_name": pytestconfig.getoption("--cluster_pool_name"),
+ }
diff --git a/boilerplate/flyte/end2end/functional-test-config.yaml b/boilerplate/flyte/end2end/functional-test-config.yaml
new file mode 100644
index 00000000000..13fc4456759
--- /dev/null
+++ b/boilerplate/flyte/end2end/functional-test-config.yaml
@@ -0,0 +1,5 @@
+admin:
+ # For GRPC endpoints you might want to use dns:///flyte.myexample.com
+ endpoint: dns:///localhost:30080
+ authType: Pkce
+ insecure: true
diff --git a/boilerplate/flyte/end2end/test_run.py b/boilerplate/flyte/end2end/test_run.py
new file mode 100644
index 00000000000..b300ee974a6
--- /dev/null
+++ b/boilerplate/flyte/end2end/test_run.py
@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+import json
+import sys
+import time
+import traceback
+from typing import Dict, List, Optional
+
+import pytest
+import requests
+from flytekit.configuration import Config
+from flytekit.models.core.execution import WorkflowExecutionPhase
+from flytekit.remote import FlyteRemote
+from flytekit.remote.executions import FlyteWorkflowExecution
+
+WAIT_TIME = 10
+MAX_ATTEMPTS = 200
+
+def execute_workflow(
+ remote: FlyteRemote,
+ version,
+ workflow_name,
+ inputs,
+ cluster_pool_name: Optional[str] = None,
+):
+ print(f"Fetching workflow={workflow_name} and version={version}")
+ wf = remote.fetch_workflow(name=workflow_name, version=version)
+ return remote.execute(wf, inputs=inputs, wait=False, cluster_pool=cluster_pool_name)
+
+def executions_finished(
+ executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]]
+) -> bool:
+ for executions in executions_by_wfgroup.values():
+ if not all([execution.is_done for execution in executions]):
+ return False
+ return True
+
+def sync_executions(
+ remote: FlyteRemote, executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]]
+):
+ try:
+ for executions in executions_by_wfgroup.values():
+ for execution in executions:
+ print(f"About to sync execution_id={execution.id.name}")
+ remote.sync(execution)
+ except Exception:
+ print(traceback.format_exc())
+ print("GOT TO THE EXCEPT")
+ print("COUNT THIS!")
+
+def report_executions(executions_by_wfgroup: Dict[str, List[FlyteWorkflowExecution]]):
+ for executions in executions_by_wfgroup.values():
+ for execution in executions:
+ print(execution)
+
+def schedule_workflow_groups(
+ tag: str,
+ workflow_groups: List[str],
+ remote: FlyteRemote,
+ terminate_workflow_on_failure: bool,
+ parsed_manifest: List[dict],
+ cluster_pool_name: Optional[str] = None,
+) -> Dict[str, bool]:
+ executions_by_wfgroup = {}
+ # Schedule executions for each workflow group,
+ for wf_group in workflow_groups:
+ workflow_group_item = list(
+ filter(lambda item: item["name"] == wf_group, parsed_manifest)
+ )
+ if not workflow_group_item:
+ continue
+ workflows = workflow_group_item[0].get("examples")
+ if not workflows:
+ continue
+ executions_by_wfgroup[wf_group] = [
+ execute_workflow(remote, tag, workflow[0], workflow[1], cluster_pool_name)
+ for workflow in workflows
+ ]
+
+ # Wait for all executions to finish
+ attempt = 0
+ while attempt == 0 or (
+ not executions_finished(executions_by_wfgroup) and attempt < MAX_ATTEMPTS
+ ):
+ attempt += 1
+ print(
+ f"Not all executions finished yet. Sleeping for some time, will check again in {WAIT_TIME}s"
+ )
+ time.sleep(WAIT_TIME)
+ sync_executions(remote, executions_by_wfgroup)
+
+ report_executions(executions_by_wfgroup)
+
+ results = {}
+ for wf_group, executions in executions_by_wfgroup.items():
+ non_succeeded_executions = []
+ for execution in executions:
+ if execution.closure.phase != WorkflowExecutionPhase.SUCCEEDED:
+ non_succeeded_executions.append(execution)
+ # Report failing cases
+ if len(non_succeeded_executions) != 0:
+ print(f"Failed executions for {wf_group}:")
+ for execution in non_succeeded_executions:
+ print(
+ f" workflow={execution.spec.launch_plan.name}, execution_id={execution.id.name}"
+ )
+ if terminate_workflow_on_failure:
+ remote.terminate(
+ execution, "aborting execution scheduled in functional test"
+ )
+ # A workflow group succeeds iff all of its executions succeed
+ results[wf_group] = len(non_succeeded_executions) == 0
+ return results
+
+def valid(workflow_group, parsed_manifest):
+ """
+ Return True if a workflow group is contained in parsed_manifest,
+ False otherwise.
+ """
+ return workflow_group in set(wf_group["name"] for wf_group in parsed_manifest)
+
+def test_run(setup_flytesnacks_env):
+
+ env = setup_flytesnacks_env
+
+ flytesnacks_release_tag = env["flytesnacks_release_tag"]
+ priorities = env["priorities"]
+ config_file_path = env["config_file"]
+ terminate_workflow_on_failure = env["terminate_workflow_on_failure"]
+ test_project_name = env["test_project_name"]
+ test_project_domain = env["test_project_domain"]
+ cluster_pool_name = env["cluster_pool_name"]
+ return_non_zero_on_failure = env["return_non_zero_on_failure"]
+
+ remote = FlyteRemote(
+ Config.auto(config_file=config_file_path),
+ test_project_name,
+ test_project_domain,
+ )
+
+ # For a given release tag and priority, this function filters the workflow groups from the flytesnacks
+ # manifest file. For example, for the release tag "v0.2.224" and the priority "P0" it returns [ "core" ].
+ manifest_url = (
+ "https://raw.githubusercontent.com/flyteorg/flytesnacks/"
+ f"{flytesnacks_release_tag}/flyte_tests_manifest.json"
+ )
+ r = requests.get(manifest_url)
+ parsed_manifest = r.json()
+ workflow_groups = []
+ workflow_groups = (
+ ["lite"]
+ if "lite" in priorities
+ else [
+ group["name"]
+ for group in parsed_manifest
+ if group["priority"] in priorities
+ ]
+ )
+
+ results = []
+ valid_workgroups = []
+ for workflow_group in workflow_groups:
+ if not valid(workflow_group, parsed_manifest):
+ results.append(
+ {
+ "label": workflow_group,
+ "status": "coming soon",
+ "color": "grey",
+ }
+ )
+ continue
+ valid_workgroups.append(workflow_group)
+
+ results_by_wfgroup = schedule_workflow_groups(
+ flytesnacks_release_tag,
+ valid_workgroups,
+ remote,
+ terminate_workflow_on_failure,
+ parsed_manifest,
+ cluster_pool_name,
+ )
+
+ for workflow_group, succeeded in results_by_wfgroup.items():
+ if succeeded:
+ background_color = "green"
+ status = "passing"
+ else:
+ background_color = "red"
+ status = "failing"
+
+ # Workflow groups can be only in one of three states:
+ # 1. passing: this indicates all the workflow executions for that workflow group
+ # executed successfully
+ # 2. failing: this state indicates that at least one execution failed in that
+ # workflow group
+ # 3. coming soon: this state is used to indicate that the workflow group was not
+ # implemented yet.
+ #
+ # Each state has a corresponding status and color to be used in the badge for that
+ # workflow group.
+ result = {
+ "label": workflow_group,
+ "status": status,
+ "color": background_color,
+ }
+ results.append(result)
+
+ print(f"Result of run:\n{json.dumps(results)}")
+
+ if return_non_zero_on_failure:
+ fail_results = [result for result in results if result["status"] not in ("passing", "coming soon")]
+ if fail_results:
+ fail_msgs = [
+ f"Workflow '{r['label']}' failed with status '{r['status']}'" for r in fail_results
+ ]
+ pytest.fail("\n".join(fail_msgs))
+
+ assert results == [{"label": "core", "status": "passing", "color": "green"}]
diff --git a/boilerplate/flyte/flyte_golang_compile/Readme.rst b/boilerplate/flyte/flyte_golang_compile/Readme.rst
new file mode 100644
index 00000000000..e6b56dd16ec
--- /dev/null
+++ b/boilerplate/flyte/flyte_golang_compile/Readme.rst
@@ -0,0 +1,16 @@
+Flyte Golang Compile
+~~~~~~~~~~~~~~~~~~~~
+
+Common compile script for Flyte golang services.
+
+**To Enable:**
+
+Add ``flyteorg/flyte_golang_compile`` to your ``boilerplate/update.cfg`` file.
+
+Add the following to your Makefile
+
+::
+
+ .PHONY: compile_linux
+ compile_linux:
+ PACKAGES={{ *your packages }} OUTPUT={{ /path/to/output }} ./boilerplate/flyte/flyte_golang_compile.sh
diff --git a/boilerplate/flyte/flyte_golang_compile/flyte_golang_compile.Template b/boilerplate/flyte/flyte_golang_compile/flyte_golang_compile.Template
new file mode 100644
index 00000000000..f587e971beb
--- /dev/null
+++ b/boilerplate/flyte/flyte_golang_compile/flyte_golang_compile.Template
@@ -0,0 +1,26 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+if [ -z "$PACKAGES" ]; then
+ echo "PACKAGES environment VAR not set"
+ exit 1
+fi
+
+if [ -z "$OUTPUT" ]; then
+ echo "OUTPUT environment VAR not set"
+ exit 1
+fi
+
+# get the GIT_SHA and RELEASE_SEMVER
+
+GIT_SHA=$(git rev-parse HEAD)
+RELEASE_SEMVER=$(git describe --tags --exact-match $GIT_SHA 2>/dev/null)
+
+CURRENT_PKG=github.com/flyteorg/{{ REPOSITORY }}
+VERSION_PKG="${CURRENT_PKG}/vendor/github.com/flyteorg/flytestdlib"
+
+LDFLAGS="-X ${VERSION_PKG}/version.Build=${GIT_SHA} -X ${VERSION_PKG}/version.Version=${RELEASE_SEMVER}"
+
+GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "$LDFLAGS" -o "$OUTPUT" "$PACKAGES"
diff --git a/boilerplate/flyte/flyte_golang_compile/update.sh b/boilerplate/flyte/flyte_golang_compile/update.sh
new file mode 100755
index 00000000000..b1e6101c2b9
--- /dev/null
+++ b/boilerplate/flyte/flyte_golang_compile/update.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+echo " - generating ${DIR}/flyte_golang_compile.sh"
+sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/flyte_golang_compile.Template > ${DIR}/flyte_golang_compile.sh
diff --git a/boilerplate/flyte/github_workflows/Readme.rst b/boilerplate/flyte/github_workflows/Readme.rst
new file mode 100644
index 00000000000..f236923514f
--- /dev/null
+++ b/boilerplate/flyte/github_workflows/Readme.rst
@@ -0,0 +1,22 @@
+Golang Github Actions
+~~~~~~~~~~~~~~~~~
+
+Provides a two github actions workflows.
+
+**To Enable:**
+
+Add ``flyteorg/github_workflows`` to your ``boilerplate/update.cfg`` file.
+
+Add a github secret ``package_name`` with the name to use for publishing (e.g. ``flytepropeller``). Typically, this will be the same name as the repository.
+
+*Note*: If you are working on a fork, include that prefix in your package name (``myfork/flytepropeller``).
+
+The actions will push to 2 repos:
+
+ 1. ``docker.pkg.github.com/flyteorg//``
+ 2. ``docker.pkg.github.com/flyteorg//-stages`` : this repo is used to cache build stages to speed up iterative builds after.
+
+There are two workflows that get deployed:
+
+ 1. A workflow that runs on Pull Requests to build and push images to github registry tagged with the commit sha.
+ 2. A workflow that runs on master merges that bump the patch version of release tag, builds and pushes images to github registry tagged with the version, commit sha as well as "latest"
diff --git a/boilerplate/flyte/github_workflows/boilerplate_automation.yml b/boilerplate/flyte/github_workflows/boilerplate_automation.yml
new file mode 100644
index 00000000000..4e586a2e7e6
--- /dev/null
+++ b/boilerplate/flyte/github_workflows/boilerplate_automation.yml
@@ -0,0 +1,36 @@
+name: Update Boilerplate Automation
+on:
+ workflow_dispatch:
+jobs:
+ update-boilerplate:
+ name: Update Boilerplate
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: "0"
+ - name: Update Boilerplate
+ run: |
+ make update_boilerplate
+ - name: Create Pull Request
+ id: cpr
+ uses: peter-evans/create-pull-request@v3
+ with:
+ token: ${{ secrets.FLYTE_BOT_PAT }}
+ commit-message: Update Boilerplate
+ committer: Flyte-Bot
+ author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+ signoff: true
+ branch: flyte-bot-update-boilerplate
+ delete-branch: true
+ title: 'Update Boilerplate'
+ body: |
+ Update Boilerplate
+ - Auto-generated by [flyte-bot]
+ labels: |
+ boilerplate
+ team-reviewers: |
+ owners
+ maintainers
+ draft: false
+
diff --git a/boilerplate/flyte/github_workflows/master.yml b/boilerplate/flyte/github_workflows/master.yml
new file mode 100644
index 00000000000..8007b291d0b
--- /dev/null
+++ b/boilerplate/flyte/github_workflows/master.yml
@@ -0,0 +1,31 @@
+name: Master
+
+on:
+ push:
+ branches:
+ - master
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: '0'
+ - name: Bump version and push tag
+ id: bump-version
+ uses: anothrNick/github-tag-action@1.17.2
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ WITH_V: true
+ DEFAULT_BUMP: patch
+ - name: Push Docker Image to Github Registry
+ uses: whoan/docker-build-with-cache-action@v5
+ with:
+ username: "${{ github.actor }}"
+ password: "${{ secrets.GITHUB_TOKEN }}"
+ image_name: ${{ secrets.package_name }}
+ image_tag: latest,${{ github.sha }},${{ steps.bump-version.outputs.tag }}
+ push_git_tag: true
+ registry: docker.pkg.github.com
+ build_extra_args: "--compress=true"
diff --git a/boilerplate/flyte/github_workflows/pull_request.yml b/boilerplate/flyte/github_workflows/pull_request.yml
new file mode 100644
index 00000000000..f4e9a5252a4
--- /dev/null
+++ b/boilerplate/flyte/github_workflows/pull_request.yml
@@ -0,0 +1,19 @@
+name: Pull Request
+
+on:
+ pull_request
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - name: Push Docker Image to Github Registry
+ uses: whoan/docker-build-with-cache-action@v5
+ with:
+ username: "${{ github.actor }}"
+ password: "${{ secrets.GITHUB_TOKEN }}"
+ image_name: ${{ secrets.package_name }}
+ image_tag: ${{ github.sha }}
+ push_git_tag: true
+ registry: docker.pkg.github.com
diff --git a/boilerplate/flyte/github_workflows/stale.yml b/boilerplate/flyte/github_workflows/stale.yml
new file mode 100644
index 00000000000..385321acb95
--- /dev/null
+++ b/boilerplate/flyte/github_workflows/stale.yml
@@ -0,0 +1,61 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 120
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: 7
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+ - pinned
+ - security
+ - "[Status] Maybe Later"
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking as stale
+staleLabel: wontfix
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+ This issue/pullrequest has been automatically marked as stale because it has not had
+ recent activity. It will be closed if no further activity occurs. Thank you
+ for your contributions.
+
+# Comment to post when removing the stale label.
+# unmarkComment: >
+# Your comment here.
+
+# Comment to post when closing a stale Issue or Pull Request.
+# closeComment: >
+# Your comment here.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 30
+
+# Limit to only `issues` or `pulls`
+only: pulls
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+# daysUntilStale: 30
+# markComment: >
+# This pull request has been automatically marked as stale because it has not had
+# recent activity. It will be closed if no further activity occurs. Thank you
+# for your contributions.
+
+# issues:
+# exemptLabels:
+# - confirmed
\ No newline at end of file
diff --git a/boilerplate/flyte/github_workflows/update.sh b/boilerplate/flyte/github_workflows/update.sh
new file mode 100755
index 00000000000..15c3cb18e2b
--- /dev/null
+++ b/boilerplate/flyte/github_workflows/update.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+mkdir -p ${DIR}/../../../.github/workflows
+
+echo " - generating github action workflows in root directory."
+sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/master.yml > ${DIR}/../../../.github/workflows/master.yml
+sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/pull_request.yml > ${DIR}/../../../.github/workflows/pull_request.yml
+cp ${DIR}/stale.yml ${DIR}/../../../.github/stale.yml
diff --git a/boilerplate/flyte/golang_support_tools/go.mod b/boilerplate/flyte/golang_support_tools/go.mod
new file mode 100644
index 00000000000..9ab99b8f207
--- /dev/null
+++ b/boilerplate/flyte/golang_support_tools/go.mod
@@ -0,0 +1,284 @@
+module github.com/flyteorg/boilerplate
+
+go 1.25.0
+
+require (
+ github.com/alvaroloes/enumer v1.1.2
+ github.com/flyteorg/flyte/flytestdlib v1.11.0
+ github.com/golangci/golangci-lint v1.61.0
+ github.com/pseudomuto/protoc-gen-doc v1.4.1
+ github.com/vektra/mockery/v3 v3.7.0
+)
+
+require (
+ 4d63.com/gocheckcompilerdirectives v1.2.1 // indirect
+ 4d63.com/gochecknoglobals v0.2.1 // indirect
+ cloud.google.com/go v0.115.1 // indirect
+ cloud.google.com/go/auth v0.9.3 // indirect
+ cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
+ cloud.google.com/go/compute/metadata v0.5.0 // indirect
+ cloud.google.com/go/iam v1.2.0 // indirect
+ cloud.google.com/go/storage v1.43.0 // indirect
+ github.com/4meepo/tagalign v1.3.4 // indirect
+ github.com/Abirdcfly/dupword v0.1.1 // indirect
+ github.com/Antonboom/errname v0.1.13 // indirect
+ github.com/Antonboom/nilnil v0.1.9 // indirect
+ github.com/Antonboom/testifylint v1.4.3 // indirect
+ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 // indirect
+ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect
+ github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
+ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 // indirect
+ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+ github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect
+ github.com/Crocmagnon/fatcontext v0.5.2 // indirect
+ github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect
+ github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.0 // indirect
+ github.com/Masterminds/semver v1.5.0 // indirect
+ github.com/Masterminds/semver/v3 v3.3.0 // indirect
+ github.com/Masterminds/sprig v2.15.0+incompatible // indirect
+ github.com/OpenPeeDeeP/depguard/v2 v2.2.0 // indirect
+ github.com/alecthomas/go-check-sumtype v0.1.4 // indirect
+ github.com/alexkohler/nakedret/v2 v2.0.4 // indirect
+ github.com/alexkohler/prealloc v1.0.0 // indirect
+ github.com/alingse/asasalint v0.0.11 // indirect
+ github.com/aokoli/goutils v1.0.1 // indirect
+ github.com/ashanbrown/forbidigo v1.6.0 // indirect
+ github.com/ashanbrown/makezero v1.1.1 // indirect
+ github.com/aws/aws-sdk-go v1.44.2 // indirect
+ github.com/beorn7/perks v1.0.1 // indirect
+ github.com/bkielbasa/cyclop v1.2.1 // indirect
+ github.com/blizzy78/varnamelen v0.8.0 // indirect
+ github.com/bombsimon/wsl/v4 v4.4.1 // indirect
+ github.com/breml/bidichk v0.2.7 // indirect
+ github.com/breml/errchkjson v0.3.6 // indirect
+ github.com/butuzov/ireturn v0.3.0 // indirect
+ github.com/butuzov/mirror v1.2.0 // indirect
+ github.com/catenacyber/perfsprint v0.7.1 // indirect
+ github.com/ccojocar/zxcvbn-go v1.0.2 // indirect
+ github.com/cespare/xxhash v1.1.0 // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
+ github.com/charithe/durationcheck v0.0.10 // indirect
+ github.com/chavacava/garif v0.1.0 // indirect
+ github.com/chigopher/pathlib v0.19.1 // indirect
+ github.com/ckaznocha/intrange v0.2.0 // indirect
+ github.com/coocood/freecache v1.1.1 // indirect
+ github.com/curioswitch/go-reassign v0.2.0 // indirect
+ github.com/daixiang0/gci v0.13.5 // indirect
+ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+ github.com/denis-tingaikin/go-header v0.5.0 // indirect
+ github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+ github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect
+ github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607 // indirect
+ github.com/ettle/strcase v0.2.0 // indirect
+ github.com/evanphx/json-patch/v5 v5.6.0 // indirect
+ github.com/fatih/color v1.17.0 // indirect
+ github.com/fatih/structtag v1.2.0 // indirect
+ github.com/felixge/httpsnoop v1.0.4 // indirect
+ github.com/firefart/nonamedreturns v1.0.5 // indirect
+ github.com/flyteorg/stow v0.3.12 // indirect
+ github.com/fsnotify/fsnotify v1.9.0 // indirect
+ github.com/fzipp/gocyclo v0.6.0 // indirect
+ github.com/ghodss/yaml v1.0.0 // indirect
+ github.com/ghostiam/protogetter v0.3.6 // indirect
+ github.com/go-critic/go-critic v0.11.4 // indirect
+ github.com/go-logr/logr v1.4.2 // indirect
+ github.com/go-logr/stdr v1.2.2 // indirect
+ github.com/go-openapi/jsonpointer v0.19.6 // indirect
+ github.com/go-openapi/jsonreference v0.20.2 // indirect
+ github.com/go-openapi/swag v0.22.3 // indirect
+ github.com/go-toolsmith/astcast v1.1.0 // indirect
+ github.com/go-toolsmith/astcopy v1.1.0 // indirect
+ github.com/go-toolsmith/astequal v1.2.0 // indirect
+ github.com/go-toolsmith/astfmt v1.1.0 // indirect
+ github.com/go-toolsmith/astp v1.1.0 // indirect
+ github.com/go-toolsmith/strparse v1.1.0 // indirect
+ github.com/go-toolsmith/typep v1.1.0 // indirect
+ github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
+ github.com/go-xmlfmt/xmlfmt v1.1.2 // indirect
+ github.com/gobwas/glob v0.2.3 // indirect
+ github.com/gofrs/flock v0.12.1 // indirect
+ github.com/gogo/protobuf v1.3.2 // indirect
+ github.com/golang-jwt/jwt/v5 v5.2.2 // indirect
+ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+ github.com/golang/protobuf v1.5.4 // indirect
+ github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a // indirect
+ github.com/golangci/gofmt v0.0.0-20240816233607-d8596aa466a9 // indirect
+ github.com/golangci/misspell v0.6.0 // indirect
+ github.com/golangci/modinfo v0.3.4 // indirect
+ github.com/golangci/plugin-module-register v0.1.1 // indirect
+ github.com/golangci/revgrep v0.5.3 // indirect
+ github.com/golangci/unconvert v0.0.0-20240309020433-c5143eacb3ed // indirect
+ github.com/google/gnostic-models v0.6.8 // indirect
+ github.com/google/go-cmp v0.6.0 // indirect
+ github.com/google/gofuzz v1.2.0 // indirect
+ github.com/google/s2a-go v0.1.8 // indirect
+ github.com/google/uuid v1.6.0 // indirect
+ github.com/googleapis/enterprise-certificate-proxy v0.3.3 // indirect
+ github.com/googleapis/gax-go/v2 v2.13.0 // indirect
+ github.com/gordonklaus/ineffassign v0.1.0 // indirect
+ github.com/gostaticanalysis/analysisutil v0.7.1 // indirect
+ github.com/gostaticanalysis/comment v1.4.2 // indirect
+ github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect
+ github.com/gostaticanalysis/nilerr v0.1.1 // indirect
+ github.com/hashicorp/go-version v1.7.0 // indirect
+ github.com/hashicorp/hcl v1.0.0 // indirect
+ github.com/hexops/gotextdiff v1.0.3 // indirect
+ github.com/huandu/xstrings v1.5.0 // indirect
+ github.com/iancoleman/strcase v0.3.0 // indirect
+ github.com/imdario/mergo v0.3.6 // indirect
+ github.com/inconshreveable/mousetrap v1.1.0 // indirect
+ github.com/jgautheron/goconst v1.7.1 // indirect
+ github.com/jingyugao/rowserrcheck v1.1.1 // indirect
+ github.com/jinzhu/copier v0.3.5 // indirect
+ github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect
+ github.com/jjti/go-spancheck v0.6.2 // indirect
+ github.com/jmespath/go-jmespath v0.4.0 // indirect
+ github.com/josharian/intern v1.0.0 // indirect
+ github.com/json-iterator/go v1.1.12 // indirect
+ github.com/julz/importas v0.1.0 // indirect
+ github.com/karamaru-alpha/copyloopvar v1.1.0 // indirect
+ github.com/kisielk/errcheck v1.7.0 // indirect
+ github.com/kkHAIKE/contextcheck v1.1.5 // indirect
+ github.com/kulti/thelper v0.6.3 // indirect
+ github.com/kunwardeep/paralleltest v1.0.10 // indirect
+ github.com/kylelemons/godebug v1.1.0 // indirect
+ github.com/kyoh86/exportloopref v0.1.11 // indirect
+ github.com/lasiar/canonicalheader v1.1.1 // indirect
+ github.com/ldez/gomoddirectives v0.2.4 // indirect
+ github.com/ldez/tagliatelle v0.5.0 // indirect
+ github.com/leonklingele/grouper v1.1.2 // indirect
+ github.com/lufeee/execinquery v1.2.1 // indirect
+ github.com/macabu/inamedparam v0.1.3 // indirect
+ github.com/magiconair/properties v1.8.7 // indirect
+ github.com/mailru/easyjson v0.7.7 // indirect
+ github.com/maratori/testableexamples v1.0.0 // indirect
+ github.com/maratori/testpackage v1.1.1 // indirect
+ github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26 // indirect
+ github.com/mattn/go-colorable v0.1.14 // indirect
+ github.com/mattn/go-isatty v0.0.20 // indirect
+ github.com/mattn/go-runewidth v0.0.19 // indirect
+ github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+ github.com/mgechev/revive v1.3.9 // indirect
+ github.com/mitchellh/go-homedir v1.1.0 // indirect
+ github.com/mitchellh/mapstructure v1.5.0 // indirect
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+ github.com/modern-go/reflect2 v1.0.2 // indirect
+ github.com/moricho/tparallel v0.3.2 // indirect
+ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+ github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007 // indirect
+ github.com/nakabonne/nestif v0.3.1 // indirect
+ github.com/ncw/swift v1.0.53 // indirect
+ github.com/nishanths/exhaustive v0.12.0 // indirect
+ github.com/nishanths/predeclared v0.2.2 // indirect
+ github.com/nunnatsa/ginkgolinter v0.16.2 // indirect
+ github.com/olekukonko/tablewriter v0.0.5 // indirect
+ github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 // indirect
+ github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+ github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
+ github.com/pkg/errors v0.9.1 // indirect
+ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+ github.com/polyfloyd/go-errorlint v1.6.0 // indirect
+ github.com/prometheus/client_golang v1.16.0 // indirect
+ github.com/prometheus/client_model v0.4.0 // indirect
+ github.com/prometheus/common v0.44.0 // indirect
+ github.com/prometheus/procfs v0.10.1 // indirect
+ github.com/pseudomuto/protokit v0.2.0 // indirect
+ github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1 // indirect
+ github.com/quasilyte/go-ruleguard/dsl v0.3.22 // indirect
+ github.com/quasilyte/gogrep v0.5.0 // indirect
+ github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect
+ github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect
+ github.com/rs/zerolog v1.34.0 // indirect
+ github.com/ryancurrah/gomodguard v1.3.5 // indirect
+ github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect
+ github.com/sanposhiho/wastedassign/v2 v2.0.7 // indirect
+ github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
+ github.com/sashamelentyev/interfacebloat v1.1.0 // indirect
+ github.com/sashamelentyev/usestdlibvars v1.27.0 // indirect
+ github.com/securego/gosec/v2 v2.21.2 // indirect
+ github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect
+ github.com/sirupsen/logrus v1.9.3 // indirect
+ github.com/sivchari/containedctx v1.0.3 // indirect
+ github.com/sivchari/tenv v1.10.0 // indirect
+ github.com/sonatard/noctx v0.0.2 // indirect
+ github.com/sourcegraph/go-diff v0.7.0 // indirect
+ github.com/spf13/afero v1.11.0 // indirect
+ github.com/spf13/cast v1.5.0 // indirect
+ github.com/spf13/cobra v1.10.2 // indirect
+ github.com/spf13/jwalterweatherman v1.1.0 // indirect
+ github.com/spf13/pflag v1.0.10 // indirect
+ github.com/spf13/viper v1.15.0 // indirect
+ github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect
+ github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect
+ github.com/stretchr/objx v0.5.3 // indirect
+ github.com/stretchr/testify v1.11.1 // indirect
+ github.com/subosito/gotenv v1.4.2 // indirect
+ github.com/tdakkota/asciicheck v0.2.0 // indirect
+ github.com/tetafro/godot v1.4.17 // indirect
+ github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 // indirect
+ github.com/timonwong/loggercheck v0.9.4 // indirect
+ github.com/tomarrell/wrapcheck/v2 v2.9.0 // indirect
+ github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect
+ github.com/ultraware/funlen v0.1.0 // indirect
+ github.com/ultraware/whitespace v0.1.1 // indirect
+ github.com/uudashr/gocognit v1.1.3 // indirect
+ github.com/xen0n/gosmopolitan v1.2.2 // indirect
+ github.com/yagipy/maintidx v1.0.0 // indirect
+ github.com/yeya24/promlinter v0.3.0 // indirect
+ github.com/ykadowak/zerologlint v0.1.5 // indirect
+ gitlab.com/bosi/decorder v0.4.2 // indirect
+ go-simpler.org/musttag v0.12.2 // indirect
+ go-simpler.org/sloglint v0.7.2 // indirect
+ go.opencensus.io v0.24.0 // indirect
+ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
+ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
+ go.opentelemetry.io/otel v1.29.0 // indirect
+ go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect
+ go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.19.0 // indirect
+ go.opentelemetry.io/otel/metric v1.29.0 // indirect
+ go.opentelemetry.io/otel/sdk v1.28.0 // indirect
+ go.opentelemetry.io/otel/trace v1.29.0 // indirect
+ go.uber.org/automaxprocs v1.5.3 // indirect
+ go.uber.org/multierr v1.11.0 // indirect
+ go.uber.org/zap v1.25.0 // indirect
+ golang.org/x/crypto v0.48.0 // indirect
+ golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect
+ golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect
+ golang.org/x/mod v0.33.0 // indirect
+ golang.org/x/net v0.50.0 // indirect
+ golang.org/x/oauth2 v0.27.0 // indirect
+ golang.org/x/sync v0.19.0 // indirect
+ golang.org/x/sys v0.41.0 // indirect
+ golang.org/x/term v0.40.0 // indirect
+ golang.org/x/text v0.34.0 // indirect
+ golang.org/x/time v0.6.0 // indirect
+ golang.org/x/tools v0.42.0 // indirect
+ google.golang.org/api v0.196.0 // indirect
+ google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect
+ google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
+ google.golang.org/grpc v1.66.0 // indirect
+ google.golang.org/protobuf v1.34.2 // indirect
+ gopkg.in/inf.v0 v0.9.1 // indirect
+ gopkg.in/ini.v1 v1.67.0 // indirect
+ gopkg.in/yaml.v2 v2.4.0 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
+ honnef.co/go/tools v0.5.1 // indirect
+ k8s.io/api v0.28.2 // indirect
+ k8s.io/apimachinery v0.28.2 // indirect
+ k8s.io/client-go v0.28.1 // indirect
+ k8s.io/klog/v2 v2.100.1 // indirect
+ k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
+ k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
+ mvdan.cc/gofumpt v0.7.0 // indirect
+ mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f // indirect
+ sigs.k8s.io/controller-runtime v0.0.0-00010101000000-000000000000 // indirect
+ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+ sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+ sigs.k8s.io/yaml v1.3.0 // indirect
+)
+
+replace (
+ github.com/pseudomuto/protoc-gen-doc => github.com/flyteorg/protoc-gen-doc v1.4.2
+ sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.16.2
+)
diff --git a/boilerplate/flyte/golang_support_tools/go.sum b/boilerplate/flyte/golang_support_tools/go.sum
new file mode 100644
index 00000000000..ab864b6c106
--- /dev/null
+++ b/boilerplate/flyte/golang_support_tools/go.sum
@@ -0,0 +1,881 @@
+4d63.com/gocheckcompilerdirectives v1.2.1 h1:AHcMYuw56NPjq/2y615IGg2kYkBdTvOaojYCBcRE7MA=
+4d63.com/gocheckcompilerdirectives v1.2.1/go.mod h1:yjDJSxmDTtIHHCqX0ufRYZDL6vQtMG7tJdKVeWwsqvs=
+4d63.com/gochecknoglobals v0.2.1 h1:1eiorGsgHOFOuoOiJDy2psSrQbRdIHrlge0IJIkUgDc=
+4d63.com/gochecknoglobals v0.2.1/go.mod h1:KRE8wtJB3CXCsb1xy421JfTHIIbmT3U5ruxw2Qu8fSU=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.115.1 h1:Jo0SM9cQnSkYfp44+v+NQXHpcHqlnRJk2qxh6yvxxxQ=
+cloud.google.com/go v0.115.1/go.mod h1:DuujITeaufu3gL68/lOFIirVNJwQeyf5UXyi+Wbgknc=
+cloud.google.com/go/auth v0.9.3 h1:VOEUIAADkkLtyfr3BLa3R8Ed/j6w1jTBmARx+wb5w5U=
+cloud.google.com/go/auth v0.9.3/go.mod h1:7z6VY+7h3KUdRov5F1i8NDP5ZzWKYmEPO842BgCsmTk=
+cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY=
+cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc=
+cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY=
+cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
+cloud.google.com/go/iam v1.2.0 h1:kZKMKVNk/IsSSc/udOb83K0hL/Yh/Gcqpz+oAkoIFN8=
+cloud.google.com/go/iam v1.2.0/go.mod h1:zITGuWgsLZxd8OwAlX+eMFgZDXzBm7icj1PVTYG766Q=
+cloud.google.com/go/longrunning v0.6.0 h1:mM1ZmaNsQsnb+5n1DNPeL0KwQd9jQRqSqSDEkBZr+aI=
+cloud.google.com/go/longrunning v0.6.0/go.mod h1:uHzSZqW89h7/pasCWNYdUpwGz3PcVWhrWupreVPYLts=
+cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs=
+cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0=
+github.com/4meepo/tagalign v1.3.4 h1:P51VcvBnf04YkHzjfclN6BbsopfJR5rxs1n+5zHt+w8=
+github.com/4meepo/tagalign v1.3.4/go.mod h1:M+pnkHH2vG8+qhE5bVc/zeP7HS/j910Fwa9TUSyZVI0=
+github.com/Abirdcfly/dupword v0.1.1 h1:Bsxe0fIw6OwBtXMIncaTxCLHYO5BB+3mcsR5E8VXloY=
+github.com/Abirdcfly/dupword v0.1.1/go.mod h1:B49AcJdTYYkpd4HjgAcutNGG9HZ2JWwKunH9Y2BA6sM=
+github.com/Antonboom/errname v0.1.13 h1:JHICqsewj/fNckzrfVSe+T33svwQxmjC+1ntDsHOVvM=
+github.com/Antonboom/errname v0.1.13/go.mod h1:uWyefRYRN54lBg6HseYCFhs6Qjcy41Y3Jl/dVhA87Ns=
+github.com/Antonboom/nilnil v0.1.9 h1:eKFMejSxPSA9eLSensFmjW2XTgTwJMjZ8hUHtV4s/SQ=
+github.com/Antonboom/nilnil v0.1.9/go.mod h1:iGe2rYwCq5/Me1khrysB4nwI7swQvjclR8/YRPl5ihQ=
+github.com/Antonboom/testifylint v1.4.3 h1:ohMt6AHuHgttaQ1xb6SSnxCeK4/rnK7KKzbvs7DmEck=
+github.com/Antonboom/testifylint v1.4.3/go.mod h1:+8Q9+AOLsz5ZiQiiYujJKs9mNz398+M6UgslP4qgJLA=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 h1:GJHeeA2N7xrG3q30L2UXDyuWRzDM900/65j70wcM4Ww=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 h1:Be6KInmFEKV81c0pOAEbRYehLMwmmGI1exuFj248AMk=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0/go.mod h1:WCPBHsOXfBVnivScjs2ypRfimjEW0qPVLGgJkZlrIOA=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c h1:pxW6RcqyfI9/kWtOwnv/G+AzdKuy2ZrqINhenH4HyNs=
+github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/Crocmagnon/fatcontext v0.5.2 h1:vhSEg8Gqng8awhPju2w7MKHqMlg4/NI+gSDHtR3xgwA=
+github.com/Crocmagnon/fatcontext v0.5.2/go.mod h1:87XhRMaInHP44Q7Tlc7jkgKKB7kZAOPiDkFMdKCC+74=
+github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 h1:sHglBQTwgx+rWPdisA5ynNEsoARbiCBOyGcJM4/OzsM=
+github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
+github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.0 h1:/fTUt5vmbkAcMBt4YQiuC23cV0kEsN1MVMNqeOW43cU=
+github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.0/go.mod h1:ONJg5sxcbsdQQ4pOW8TGdTidT2TMAUy/2Xhr8mrYaao=
+github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0=
+github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
+github.com/Masterminds/sprig v2.15.0+incompatible h1:0gSxPGWS9PAr7U2NsQ2YQg6juRDINkUyuvbb4b2Xm8w=
+github.com/Masterminds/sprig v2.15.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
+github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/OpenPeeDeeP/depguard/v2 v2.2.0 h1:vDfG60vDtIuf0MEOhmLlLLSzqaRM8EMcgJPdp74zmpA=
+github.com/OpenPeeDeeP/depguard/v2 v2.2.0/go.mod h1:CIzddKRvLBC4Au5aYP/i3nyaWQ+ClszLIuVocRiCYFQ=
+github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk=
+github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
+github.com/alecthomas/go-check-sumtype v0.1.4 h1:WCvlB3l5Vq5dZQTFmodqL2g68uHiSwwlWcT5a2FGK0c=
+github.com/alecthomas/go-check-sumtype v0.1.4/go.mod h1:WyYPfhfkdhyrdaligV6svFopZV8Lqdzn5pyVBaV6jhQ=
+github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
+github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/alexkohler/nakedret/v2 v2.0.4 h1:yZuKmjqGi0pSmjGpOC016LtPJysIL0WEUiaXW5SUnNg=
+github.com/alexkohler/nakedret/v2 v2.0.4/go.mod h1:bF5i0zF2Wo2o4X4USt9ntUWve6JbFv02Ff4vlkmS/VU=
+github.com/alexkohler/prealloc v1.0.0 h1:Hbq0/3fJPQhNkN0dR95AVrr6R7tou91y0uHG5pOcUuw=
+github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE=
+github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQRnw=
+github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I=
+github.com/alvaroloes/enumer v1.1.2 h1:5khqHB33TZy1GWCO/lZwcroBFh7u+0j40T83VUbfAMY=
+github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
+github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg=
+github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ=
+github.com/ashanbrown/forbidigo v1.6.0 h1:D3aewfM37Yb3pxHujIPSpTf6oQk9sc9WZi8gerOIVIY=
+github.com/ashanbrown/forbidigo v1.6.0/go.mod h1:Y8j9jy9ZYAEHXdu723cUlraTqbzjKF1MUyfOKL+AjcU=
+github.com/ashanbrown/makezero v1.1.1 h1:iCQ87C0V0vSyO+M9E/FZYbu65auqH0lnsOkf5FcB28s=
+github.com/ashanbrown/makezero v1.1.1/go.mod h1:i1bJLCRSCHOcOa9Y6MyF2FTfMZMFdHvxKHxgO5Z1axI=
+github.com/aws/aws-sdk-go v1.44.2 h1:5VBk5r06bgxgRKVaUtm1/4NT/rtrnH2E4cnAYv5zgQc=
+github.com/aws/aws-sdk-go v1.44.2/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
+github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bkielbasa/cyclop v1.2.1 h1:AeF71HZDob1P2/pRm1so9cd1alZnrpyc4q2uP2l0gJY=
+github.com/bkielbasa/cyclop v1.2.1/go.mod h1:K/dT/M0FPAiYjBgQGau7tz+3TMh4FWAEqlMhzFWCrgM=
+github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M=
+github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k=
+github.com/bombsimon/wsl/v4 v4.4.1 h1:jfUaCkN+aUpobrMO24zwyAMwMAV5eSziCkOKEauOLdw=
+github.com/bombsimon/wsl/v4 v4.4.1/go.mod h1:Xu/kDxGZTofQcDGCtQe9KCzhHphIe0fDuyWTxER9Feo=
+github.com/breml/bidichk v0.2.7 h1:dAkKQPLl/Qrk7hnP6P+E0xOodrq8Us7+U0o4UBOAlQY=
+github.com/breml/bidichk v0.2.7/go.mod h1:YodjipAGI9fGcYM7II6wFvGhdMYsC5pHDlGzqvEW3tQ=
+github.com/breml/errchkjson v0.3.6 h1:VLhVkqSBH96AvXEyclMR37rZslRrY2kcyq+31HCsVrA=
+github.com/breml/errchkjson v0.3.6/go.mod h1:jhSDoFheAF2RSDOlCfhHO9KqhZgAYLyvHe7bRCX8f/U=
+github.com/butuzov/ireturn v0.3.0 h1:hTjMqWw3y5JC3kpnC5vXmFJAWI/m31jaCYQqzkS6PL0=
+github.com/butuzov/ireturn v0.3.0/go.mod h1:A09nIiwiqzN/IoVo9ogpa0Hzi9fex1kd9PSD6edP5ZA=
+github.com/butuzov/mirror v1.2.0 h1:9YVK1qIjNspaqWutSv8gsge2e/Xpq1eqEkslEUHy5cs=
+github.com/butuzov/mirror v1.2.0/go.mod h1:DqZZDtzm42wIAIyHXeN8W/qb1EPlb9Qn/if9icBOpdQ=
+github.com/catenacyber/perfsprint v0.7.1 h1:PGW5G/Kxn+YrN04cRAZKC+ZuvlVwolYMrIyyTJ/rMmc=
+github.com/catenacyber/perfsprint v0.7.1/go.mod h1:/wclWYompEyjUD2FuIIDVKNkqz7IgBIWXIH3V0Zol50=
+github.com/ccojocar/zxcvbn-go v1.0.2 h1:na/czXU8RrhXO4EZme6eQJLR4PzcGsahsBOAwU6I3Vg=
+github.com/ccojocar/zxcvbn-go v1.0.2/go.mod h1:g1qkXtUSvHP8lhHp5GrSmTz6uWALGRMQdw6Qnz/hi60=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/charithe/durationcheck v0.0.10 h1:wgw73BiocdBDQPik+zcEoBG/ob8uyBHf2iyoHGPf5w4=
+github.com/charithe/durationcheck v0.0.10/go.mod h1:bCWXb7gYRysD1CU3C+u4ceO49LoGOY1C1L6uouGNreQ=
+github.com/chavacava/garif v0.1.0 h1:2JHa3hbYf5D9dsgseMKAmc/MZ109otzgNFk5s87H9Pc=
+github.com/chavacava/garif v0.1.0/go.mod h1:XMyYCkEL58DF0oyW4qDjjnPWONs2HBqYKI+UIPD+Gww=
+github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764=
+github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
+github.com/chigopher/pathlib v0.19.1 h1:RoLlUJc0CqBGwq239cilyhxPNLXTK+HXoASGyGznx5A=
+github.com/chigopher/pathlib v0.19.1/go.mod h1:tzC1dZLW8o33UQpWkNkhvPwL5n4yyFRFm/jL1YGWFvY=
+github.com/ckaznocha/intrange v0.2.0 h1:FykcZuJ8BD7oX93YbO1UY9oZtkRbp+1/kJcDjkefYLs=
+github.com/ckaznocha/intrange v0.2.0/go.mod h1:r5I7nUlAAG56xmkOpw4XVr16BXhwYTUdcuRFeevn1oE=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/coocood/freecache v1.1.1 h1:uukNF7QKCZEdZ9gAV7WQzvh0SbjwdMF6m3x3rxEkaPc=
+github.com/coocood/freecache v1.1.1/go.mod h1:OKrEjkGVoxZhyWAJoeFi5BMLUJm2Tit0kpGkIr7NGYY=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo=
+github.com/curioswitch/go-reassign v0.2.0/go.mod h1:x6OpXuWvgfQaMGks2BZybTngWjT84hqJfKoO8Tt/Roc=
+github.com/daixiang0/gci v0.13.5 h1:kThgmH1yBmZSBCh1EJVxQ7JsHpm5Oms0AMed/0LaH4c=
+github.com/daixiang0/gci v0.13.5/go.mod h1:12etP2OniiIdP4q+kjUGrC/rUagga7ODbqsom5Eo5Yk=
+github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8=
+github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY=
+github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
+github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/envoyproxy/protoc-gen-validate v0.3.0-java/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A=
+github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew=
+github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607 h1:cTavhURetDkezJCvxFggiyLeP40Mrk/TtVg2+ycw1Es=
+github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607/go.mod h1:Cg4fM0vhYWOZdgM7RIOSTRNIc8/VT7CXClC3Ni86lu4=
+github.com/ettle/strcase v0.2.0 h1:fGNiVF21fHXpX1niBgk0aROov1LagYsOwV/xqKDKR/Q=
+github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A=
+github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
+github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
+github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
+github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
+github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4=
+github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/firefart/nonamedreturns v1.0.5 h1:tM+Me2ZaXs8tfdDw3X6DOX++wMCOqzYUho6tUTYIdRA=
+github.com/firefart/nonamedreturns v1.0.5/go.mod h1:gHJjDqhGM4WyPt639SOZs+G89Ko7QKH5R5BhnO6xJhw=
+github.com/flyteorg/flyte/flytestdlib v1.11.0 h1:DxM/sf6H0ong8LIjgh0YwXK+abnGV8kWVi6EgfVCkO8=
+github.com/flyteorg/flyte/flytestdlib v1.11.0/go.mod h1:AmgNCq/tGEDwVfilW1nFtgPQn8vQ9gcDu6SNwz1YY+M=
+github.com/flyteorg/protoc-gen-doc v1.4.2 h1:Otw0F+RHaPQ8XlpzhLLgjsCMcrAIcMO01Zh+ALe3rrE=
+github.com/flyteorg/protoc-gen-doc v1.4.2/go.mod h1:exDTOVwqpp30eV/EDPFLZy3Pwr2sn6hBC1WIYH/UbIg=
+github.com/flyteorg/stow v0.3.12 h1:RRXI5RUdxaK6A46HrO0D2r14cRlW1lJRL6qyzqpVMPU=
+github.com/flyteorg/stow v0.3.12/go.mod h1:nyaBf8ZWkpHWkKIl4rqKI2uXfPx+VbL0PmEtvq4Pxkc=
+github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
+github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo=
+github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/ghostiam/protogetter v0.3.6 h1:R7qEWaSgFCsy20yYHNIJsU9ZOb8TziSRRxuAOTVKeOk=
+github.com/ghostiam/protogetter v0.3.6/go.mod h1:7lpeDnEJ1ZjL/YtyoN99ljO4z0pd3H0d18/t2dPBxHw=
+github.com/go-critic/go-critic v0.11.4 h1:O7kGOCx0NDIni4czrkRIXTnit0mkyKOCePh3My6OyEU=
+github.com/go-critic/go-critic v0.11.4/go.mod h1:2QAdo4iuLik5S9YG0rT4wcZ8QxwHYkrr6/2MWAiv/vc=
+github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo=
+github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA=
+github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
+github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
+github.com/go-toolsmith/astcast v1.1.0 h1:+JN9xZV1A+Re+95pgnMgDboWNVnIMMQXwfBwLRPgSC8=
+github.com/go-toolsmith/astcast v1.1.0/go.mod h1:qdcuFWeGGS2xX5bLM/c3U9lewg7+Zu4mr+xPwZIB4ZU=
+github.com/go-toolsmith/astcopy v1.1.0 h1:YGwBN0WM+ekI/6SS6+52zLDEf8Yvp3n2seZITCUBt5s=
+github.com/go-toolsmith/astcopy v1.1.0/go.mod h1:hXM6gan18VA1T/daUEHCFcYiW8Ai1tIwIzHY6srfEAw=
+github.com/go-toolsmith/astequal v1.0.3/go.mod h1:9Ai4UglvtR+4up+bAD4+hCj7iTo4m/OXVTSLnCyTAx4=
+github.com/go-toolsmith/astequal v1.1.0/go.mod h1:sedf7VIdCL22LD8qIvv7Nn9MuWJruQA/ysswh64lffQ=
+github.com/go-toolsmith/astequal v1.2.0 h1:3Fs3CYZ1k9Vo4FzFhwwewC3CHISHDnVUPC4x0bI2+Cw=
+github.com/go-toolsmith/astequal v1.2.0/go.mod h1:c8NZ3+kSFtFY/8lPso4v8LuJjdJiUFVnSuU3s0qrrDY=
+github.com/go-toolsmith/astfmt v1.1.0 h1:iJVPDPp6/7AaeLJEruMsBUlOYCmvg0MoCfJprsOmcco=
+github.com/go-toolsmith/astfmt v1.1.0/go.mod h1:OrcLlRwu0CuiIBp/8b5PYF9ktGVZUjlNMV634mhwuQ4=
+github.com/go-toolsmith/astp v1.1.0 h1:dXPuCl6u2llURjdPLLDxJeZInAeZ0/eZwFJmqZMnpQA=
+github.com/go-toolsmith/astp v1.1.0/go.mod h1:0T1xFGz9hicKs8Z5MfAqSUitoUYS30pDMsRVIDHs8CA=
+github.com/go-toolsmith/pkgload v1.2.2 h1:0CtmHq/02QhxcF7E9N5LIFcYFsMR5rdovfqTtRKkgIk=
+github.com/go-toolsmith/pkgload v1.2.2/go.mod h1:R2hxLNRKuAsiXCo2i5J6ZQPhnPMOVtU+f0arbFPWCus=
+github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQiyP2Bvw=
+github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ=
+github.com/go-toolsmith/typep v1.1.0 h1:fIRYDyF+JywLfqzyhdiHzRop/GQDxxNhLGQ6gFUNHus=
+github.com/go-toolsmith/typep v1.1.0/go.mod h1:fVIw+7zjdsMxDA3ITWnH1yOiw1rnTQKCsF/sk2H/qig=
+github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
+github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/go-xmlfmt/xmlfmt v1.1.2 h1:Nea7b4icn8s57fTx1M5AI4qQT5HEM3rVUO8MuE6g80U=
+github.com/go-xmlfmt/xmlfmt v1.1.2/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E=
+github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=
+github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
+github.com/golangci/gofmt v0.0.0-20240816233607-d8596aa466a9 h1:/1322Qns6BtQxUZDTAT4SdcoxknUki7IAoK4SAXr8ME=
+github.com/golangci/gofmt v0.0.0-20240816233607-d8596aa466a9/go.mod h1:Oesb/0uFAyWoaw1U1qS5zyjCg5NP9C9iwjnI4tIsXEE=
+github.com/golangci/golangci-lint v1.61.0 h1:VvbOLaRVWmyxCnUIMTbf1kDsaJbTzH20FAMXTAlQGu8=
+github.com/golangci/golangci-lint v1.61.0/go.mod h1:e4lztIrJJgLPhWvFPDkhiMwEFRrWlmFbrZea3FsJyN8=
+github.com/golangci/misspell v0.6.0 h1:JCle2HUTNWirNlDIAUO44hUsKhOFqGPoC4LZxlaSXDs=
+github.com/golangci/misspell v0.6.0/go.mod h1:keMNyY6R9isGaSAu+4Q8NMBwMPkh15Gtc8UCVoDtAWo=
+github.com/golangci/modinfo v0.3.4 h1:oU5huX3fbxqQXdfspamej74DFX0kyGLkw1ppvXoJ8GA=
+github.com/golangci/modinfo v0.3.4/go.mod h1:wytF1M5xl9u0ij8YSvhkEVPP3M5Mc7XLl1pxH3B2aUM=
+github.com/golangci/plugin-module-register v0.1.1 h1:TCmesur25LnyJkpsVrupv1Cdzo+2f7zX0H6Jkw1Ol6c=
+github.com/golangci/plugin-module-register v0.1.1/go.mod h1:TTpqoB6KkwOJMV8u7+NyXMrkwwESJLOkfl9TxR1DGFc=
+github.com/golangci/revgrep v0.5.3 h1:3tL7c1XBMtWHHqVpS5ChmiAAoe4PF/d5+ULzV9sLAzs=
+github.com/golangci/revgrep v0.5.3/go.mod h1:U4R/s9dlXZsg8uJmaR1GrloUr14D7qDl8gi2iPXJH8k=
+github.com/golangci/unconvert v0.0.0-20240309020433-c5143eacb3ed h1:IURFTjxeTfNFP0hTEi1YKjB/ub8zkpaOqFFMApi2EAs=
+github.com/golangci/unconvert v0.0.0-20240309020433-c5143eacb3ed/go.mod h1:XLXN8bNw4CGRPaqgl3bv/lhz7bsGPh4/xSaMTbo2vkQ=
+github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
+github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
+github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
+github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA=
+github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
+github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM=
+github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA=
+github.com/google/uuid v0.0.0-20161128191214-064e2069ce9c/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/enterprise-certificate-proxy v0.3.3 h1:QRje2j5GZimBzlbhGA2V2QlGNgL8G6e+wGo/+/2bWI0=
+github.com/googleapis/enterprise-certificate-proxy v0.3.3/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=
+github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s=
+github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A=
+github.com/gordonklaus/ineffassign v0.1.0 h1:y2Gd/9I7MdY1oEIt+n+rowjBNDcLQq3RsH5hwJd0f9s=
+github.com/gordonklaus/ineffassign v0.1.0/go.mod h1:Qcp2HIAYhR7mNUVSIxZww3Guk4it82ghYcEXIAk+QT0=
+github.com/gostaticanalysis/analysisutil v0.7.1 h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk=
+github.com/gostaticanalysis/analysisutil v0.7.1/go.mod h1:v21E3hY37WKMGSnbsw2S/ojApNWb6C1//mXO48CXbVc=
+github.com/gostaticanalysis/comment v1.4.1/go.mod h1:ih6ZxzTHLdadaiSnF5WY3dxUoXfXAlTaRzuaNDlSado=
+github.com/gostaticanalysis/comment v1.4.2 h1:hlnx5+S2fY9Zo9ePo4AhgYsYHbM2+eAv8m/s1JiCd6Q=
+github.com/gostaticanalysis/comment v1.4.2/go.mod h1:KLUTGDv6HOCotCH8h2erHKmpci2ZoR8VPu34YA2uzdM=
+github.com/gostaticanalysis/forcetypeassert v0.1.0 h1:6eUflI3DiGusXGK6X7cCcIgVCpZ2CiZ1Q7jl6ZxNV70=
+github.com/gostaticanalysis/forcetypeassert v0.1.0/go.mod h1:qZEedyP/sY1lTGV1uJ3VhWZ2mqag3IkWsDHVbplHXak=
+github.com/gostaticanalysis/nilerr v0.1.1 h1:ThE+hJP0fEp4zWLkWHWcRyI2Od0p7DlgYG3Uqrmrcpk=
+github.com/gostaticanalysis/nilerr v0.1.1/go.mod h1:wZYb6YI5YAxxq0i1+VJbY0s2YONW0HU0GPE3+5PWN4A=
+github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod h1:D+FIZ+7OahH3ePw/izIEeH5I06eKs1IKI4Xr64/Am3M=
+github.com/gostaticanalysis/testutil v0.4.0 h1:nhdCmubdmDF6VEatUNjgUZBJKWRqugoISdUv3PPQgHY=
+github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Repi5x3CuviD3dgAZaBU=
+github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY=
+github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
+github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
+github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
+github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI=
+github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
+github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
+github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jgautheron/goconst v1.7.1 h1:VpdAG7Ca7yvvJk5n8dMwQhfEZJh95kl/Hl9S1OI5Jkk=
+github.com/jgautheron/goconst v1.7.1/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4=
+github.com/jingyugao/rowserrcheck v1.1.1 h1:zibz55j/MJtLsjP1OF4bSdgXxwL1b+Vn7Tjzq7gFzUs=
+github.com/jingyugao/rowserrcheck v1.1.1/go.mod h1:4yvlZSDb3IyDTUZJUmpZfm2Hwok+Dtp+nu2qOq+er9c=
+github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
+github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
+github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af h1:KA9BjwUk7KlCh6S9EAGWBt1oExIUv9WyNCiRz5amv48=
+github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
+github.com/jjti/go-spancheck v0.6.2 h1:iYtoxqPMzHUPp7St+5yA8+cONdyXD3ug6KK15n7Pklk=
+github.com/jjti/go-spancheck v0.6.2/go.mod h1:+X7lvIrR5ZdUTkxFYqzJ0abr8Sb5LOo80uOhWNqIrYA=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/julz/importas v0.1.0 h1:F78HnrsjY3cR7j0etXy5+TU1Zuy7Xt08X/1aJnH5xXY=
+github.com/julz/importas v0.1.0/go.mod h1:oSFU2R4XK/P7kNBrnL/FEQlDGN1/6WoxXEjSSXO0DV0=
+github.com/karamaru-alpha/copyloopvar v1.1.0 h1:x7gNyKcC2vRBO1H2Mks5u1VxQtYvFiym7fCjIP8RPos=
+github.com/karamaru-alpha/copyloopvar v1.1.0/go.mod h1:u7CIfztblY0jZLOQZgH3oYsJzpC2A7S6u/lfgSXHy0k=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/errcheck v1.7.0 h1:+SbscKmWJ5mOK/bO1zS60F5I9WwZDWOfRsC4RwfwRV0=
+github.com/kisielk/errcheck v1.7.0/go.mod h1:1kLL+jV4e+CFfueBmI1dSK2ADDyQnlrnrY/FqKluHJQ=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kkHAIKE/contextcheck v1.1.5 h1:CdnJh63tcDe53vG+RebdpdXJTc9atMgGqdx8LXxiilg=
+github.com/kkHAIKE/contextcheck v1.1.5/go.mod h1:O930cpht4xb1YQpK+1+AgoM3mFsvxr7uyFptcnWTYUA=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kulti/thelper v0.6.3 h1:ElhKf+AlItIu+xGnI990no4cE2+XaSu1ULymV2Yulxs=
+github.com/kulti/thelper v0.6.3/go.mod h1:DsqKShOvP40epevkFrvIwkCMNYxMeTNjdWL4dqWHZ6I=
+github.com/kunwardeep/paralleltest v1.0.10 h1:wrodoaKYzS2mdNVnc4/w31YaXFtsc21PCTdvWJ/lDDs=
+github.com/kunwardeep/paralleltest v1.0.10/go.mod h1:2C7s65hONVqY7Q5Efj5aLzRCNLjw2h4eMc9EcypGjcY=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/kyoh86/exportloopref v0.1.11 h1:1Z0bcmTypkL3Q4k+IDHMWTcnCliEZcaPiIe0/ymEyhQ=
+github.com/kyoh86/exportloopref v0.1.11/go.mod h1:qkV4UF1zGl6EkF1ox8L5t9SwyeBAZ3qLMd6up458uqA=
+github.com/lasiar/canonicalheader v1.1.1 h1:wC+dY9ZfiqiPwAexUApFush/csSPXeIi4QqyxXmng8I=
+github.com/lasiar/canonicalheader v1.1.1/go.mod h1:cXkb3Dlk6XXy+8MVQnF23CYKWlyA7kfQhSw2CcZtZb0=
+github.com/ldez/gomoddirectives v0.2.4 h1:j3YjBIjEBbqZ0NKtBNzr8rtMHTOrLPeiwTkfUJZ3alg=
+github.com/ldez/gomoddirectives v0.2.4/go.mod h1:oWu9i62VcQDYp9EQ0ONTfqLNh+mDLWWDO+SO0qSQw5g=
+github.com/ldez/tagliatelle v0.5.0 h1:epgfuYt9v0CG3fms0pEgIMNPuFf/LpPIfjk4kyqSioo=
+github.com/ldez/tagliatelle v0.5.0/go.mod h1:rj1HmWiL1MiKQuOONhd09iySTEkUuE/8+5jtPYz9xa4=
+github.com/leonklingele/grouper v1.1.2 h1:o1ARBDLOmmasUaNDesWqWCIFH3u7hoFlM84YrjT3mIY=
+github.com/leonklingele/grouper v1.1.2/go.mod h1:6D0M/HVkhs2yRKRFZUoGjeDy7EZTfFBE9gl4kjmIGkA=
+github.com/lufeee/execinquery v1.2.1 h1:hf0Ems4SHcUGBxpGN7Jz78z1ppVkP/837ZlETPCEtOM=
+github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM=
+github.com/macabu/inamedparam v0.1.3 h1:2tk/phHkMlEL/1GNe/Yf6kkR/hkcUdAEY3L0hjYV1Mk=
+github.com/macabu/inamedparam v0.1.3/go.mod h1:93FLICAIk/quk7eaPPQvbzihUdn/QkGDwIZEoLtpH6I=
+github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
+github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/maratori/testableexamples v1.0.0 h1:dU5alXRrD8WKSjOUnmJZuzdxWOEQ57+7s93SLMxb2vI=
+github.com/maratori/testableexamples v1.0.0/go.mod h1:4rhjL1n20TUTT4vdh3RDqSizKLyXp7K2u6HgraZCGzE=
+github.com/maratori/testpackage v1.1.1 h1:S58XVV5AD7HADMmD0fNnziNHqKvSdDuEKdPD1rNTU04=
+github.com/maratori/testpackage v1.1.1/go.mod h1:s4gRK/ym6AMrqpOa/kEbQTV4Q4jb7WeLZzVhVVVOQMc=
+github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26 h1:gWg6ZQ4JhDfJPqlo2srm/LN17lpybq15AryXIRcWYLE=
+github.com/matoous/godox v0.0.0-20230222163458-006bad1f9d26/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
+github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE=
+github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mgechev/revive v1.3.9 h1:18Y3R4a2USSBF+QZKFQwVkBROUda7uoBlkEuBD+YD1A=
+github.com/mgechev/revive v1.3.9/go.mod h1:+uxEIr5UH0TjXWHTno3xh4u7eg6jDpXKzQccA9UGhHU=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/moricho/tparallel v0.3.2 h1:odr8aZVFA3NZrNybggMkYO3rgPRcqjeQUlBBFVxKHTI=
+github.com/moricho/tparallel v0.3.2/go.mod h1:OQ+K3b4Ln3l2TZveGCywybl68glfLEwFGqvnjok8b+U=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007 h1:28i1IjGcx8AofiB4N3q5Yls55VEaitzuEPkFJEVgGkA=
+github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007/go.mod h1:m2XC9Qq0AlmmVksL6FktJCdTYyLk7V3fKyp0sl1yWQo=
+github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U=
+github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE=
+github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks=
+github.com/ncw/swift v1.0.53/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
+github.com/nishanths/exhaustive v0.12.0 h1:vIY9sALmw6T/yxiASewa4TQcFsVYZQQRUQJhKRf3Swg=
+github.com/nishanths/exhaustive v0.12.0/go.mod h1:mEZ95wPIZW+x8kC4TgC+9YCUgiST7ecevsVDTgc2obs=
+github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk=
+github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c=
+github.com/nunnatsa/ginkgolinter v0.16.2 h1:8iLqHIZvN4fTLDC0Ke9tbSZVcyVHoBs0HIbnVSxfHJk=
+github.com/nunnatsa/ginkgolinter v0.16.2/go.mod h1:4tWRinDN1FeJgU+iJANW/kz7xKN5nYRAOfJDQUS9dOQ=
+github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
+github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
+github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4=
+github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag=
+github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=
+github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc=
+github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw=
+github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
+github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
+github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
+github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
+github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
+github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
+github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU=
+github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
+github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
+github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
+github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/polyfloyd/go-errorlint v1.6.0 h1:tftWV9DE7txiFzPpztTAwyoRLKNj9gpVm2cg8/OwcYY=
+github.com/polyfloyd/go-errorlint v1.6.0/go.mod h1:HR7u8wuP1kb1NeN1zqTd1ZMlqUKPPHF+Id4vIPvDqVw=
+github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
+github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
+github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
+github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
+github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
+github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/pseudomuto/protokit v0.2.0 h1:hlnBDcy3YEDXH7kc9gV+NLaN0cDzhDvD1s7Y6FZ8RpM=
+github.com/pseudomuto/protokit v0.2.0/go.mod h1:2PdH30hxVHsup8KpBTOXTBeMVhJZVio3Q8ViKSAXT0Q=
+github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1 h1:+Wl/0aFp0hpuHM3H//KMft64WQ1yX9LdJY64Qm/gFCo=
+github.com/quasilyte/go-ruleguard v0.4.3-0.20240823090925-0fe6f58b47b1/go.mod h1:GJLgqsLeo4qgavUoL8JeGFNS7qcisx3awV/w9eWTmNI=
+github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE=
+github.com/quasilyte/go-ruleguard/dsl v0.3.22/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
+github.com/quasilyte/gogrep v0.5.0 h1:eTKODPXbI8ffJMN+W2aE0+oL0z/nh8/5eNdiO34SOAo=
+github.com/quasilyte/gogrep v0.5.0/go.mod h1:Cm9lpz9NZjEoL1tgZ2OgeUKPIxL1meE7eo60Z6Sk+Ng=
+github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 h1:TCg2WBOl980XxGFEZSS6KlBGIV0diGdySzxATTWoqaU=
+github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0=
+github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 h1:M8mH9eK4OUR4lu7Gd+PU1fV2/qnDNfzT635KRSObncs=
+github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8A4Y+GyBgPuaQJuWiy0XYftx4Xm/y5Jqk9I6VQ=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
+github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
+github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryancurrah/gomodguard v1.3.5 h1:cShyguSwUEeC0jS7ylOiG/idnd1TpJ1LfHGpV3oJmPU=
+github.com/ryancurrah/gomodguard v1.3.5/go.mod h1:MXlEPQRxgfPQa62O8wzK3Ozbkv9Rkqr+wKjSxTdsNJE=
+github.com/ryanrolds/sqlclosecheck v0.5.1 h1:dibWW826u0P8jNLsLN+En7+RqWWTYrjCB9fJfSfdyCU=
+github.com/ryanrolds/sqlclosecheck v0.5.1/go.mod h1:2g3dUjoS6AL4huFdv6wn55WpLIDjY7ZgUR4J8HOO/XQ=
+github.com/sanposhiho/wastedassign/v2 v2.0.7 h1:J+6nrY4VW+gC9xFzUc+XjPD3g3wF3je/NsJFwFK7Uxc=
+github.com/sanposhiho/wastedassign/v2 v2.0.7/go.mod h1:KyZ0MWTwxxBmfwn33zh3k1dmsbF2ud9pAAGfoLfjhtI=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
+github.com/sashamelentyev/interfacebloat v1.1.0 h1:xdRdJp0irL086OyW1H/RTZTr1h/tMEOsumirXcOJqAw=
+github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ=
+github.com/sashamelentyev/usestdlibvars v1.27.0 h1:t/3jZpSXtRPRf2xr0m63i32ZrusyurIGT9E5wAvXQnI=
+github.com/sashamelentyev/usestdlibvars v1.27.0/go.mod h1:9nl0jgOfHKWNFS43Ojw0i7aRoS4j6EBye3YBhmAIRF8=
+github.com/securego/gosec/v2 v2.21.2 h1:deZp5zmYf3TWwU7A7cR2+SolbTpZ3HQiwFqnzQyEl3M=
+github.com/securego/gosec/v2 v2.21.2/go.mod h1:au33kg78rNseF5PwPnTWhuYBFf534bvJRvOrgZ/bFzU=
+github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=
+github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sivchari/containedctx v1.0.3 h1:x+etemjbsh2fB5ewm5FeLNi5bUjK0V8n0RB+Wwfd0XE=
+github.com/sivchari/containedctx v1.0.3/go.mod h1:c1RDvCbnJLtH4lLcYD/GqwiBSSf4F5Qk0xld2rBqzJ4=
+github.com/sivchari/tenv v1.10.0 h1:g/hzMA+dBCKqGXgW8AV/1xIWhAvDrx0zFKNR48NFMg0=
+github.com/sivchari/tenv v1.10.0/go.mod h1:tdY24masnVoZFxYrHv/nD6Tc8FbkEtAQEEziXpyMgqY=
+github.com/sonatard/noctx v0.0.2 h1:L7Dz4De2zDQhW8S0t+KUjY0MAQJd6SgVwhzNIc4ok00=
+github.com/sonatard/noctx v0.0.2/go.mod h1:kzFz+CzWSjQ2OzIm46uJZoXuBpa2+0y3T36U18dWqIo=
+github.com/sourcegraph/go-diff v0.7.0 h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0=
+github.com/sourcegraph/go-diff v0.7.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
+github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
+github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=
+github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU=
+github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
+github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
+github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU=
+github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA=
+github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YEwQ0=
+github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I=
+github.com/stbenjam/no-sprintf-host-port v0.1.1 h1:tYugd/yrm1O0dV+ThCbaKZh195Dfm07ysF0U6JQXczc=
+github.com/stbenjam/no-sprintf-host-port v0.1.1/go.mod h1:TLhvtIvONRzdmkFiio4O8LHsN9N74I+PhRquPsxpL0I=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
+github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
+github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
+github.com/tdakkota/asciicheck v0.2.0 h1:o8jvnUANo0qXtnslk2d3nMKTFNlOnJjRrNcj0j9qkHM=
+github.com/tdakkota/asciicheck v0.2.0/go.mod h1:Qb7Y9EgjCLJGup51gDHFzbI08/gbGhL/UVhYIPWG2rg=
+github.com/tenntenn/modver v1.0.1 h1:2klLppGhDgzJrScMpkj9Ujy3rXPUspSjAcev9tSEBgA=
+github.com/tenntenn/modver v1.0.1/go.mod h1:bePIyQPb7UeioSRkw3Q0XeMhYZSMx9B8ePqg6SAMGH0=
+github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3 h1:f+jULpRQGxTSkNYKJ51yaw6ChIqO+Je8UqsTKN/cDag=
+github.com/tenntenn/text/transform v0.0.0-20200319021203-7eef512accb3/go.mod h1:ON8b8w4BN/kE1EOhwT0o+d62W65a6aPw1nouo9LMgyY=
+github.com/tetafro/godot v1.4.17 h1:pGzu+Ye7ZUEFx7LHU0dAKmCOXWsPjl7qA6iMGndsjPs=
+github.com/tetafro/godot v1.4.17/go.mod h1:2oVxTBSftRTh4+MVfUaUXR6bn2GDXCaMcOG4Dk3rfio=
+github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 h1:quvGphlmUVU+nhpFa4gg4yJyTRJ13reZMDHrKwYw53M=
+github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966/go.mod h1:27bSVNWSBOHm+qRp1T9qzaIpsWEP6TbUnei/43HK+PQ=
+github.com/timonwong/loggercheck v0.9.4 h1:HKKhqrjcVj8sxL7K77beXh0adEm6DLjV/QOGeMXEVi4=
+github.com/timonwong/loggercheck v0.9.4/go.mod h1:caz4zlPcgvpEkXgVnAJGowHAMW2NwHaNlpS8xDbVhTg=
+github.com/tomarrell/wrapcheck/v2 v2.9.0 h1:801U2YCAjLhdN8zhZ/7tdjB3EnAoRlJHt/s+9hijLQ4=
+github.com/tomarrell/wrapcheck/v2 v2.9.0/go.mod h1:g9vNIyhb5/9TQgumxQyOEqDHsmGYcGsVMOx/xGkqdMo=
+github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw=
+github.com/tommy-muehle/go-mnd/v2 v2.5.1/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw=
+github.com/ultraware/funlen v0.1.0 h1:BuqclbkY6pO+cvxoq7OsktIXZpgBSkYTQtmwhAK81vI=
+github.com/ultraware/funlen v0.1.0/go.mod h1:XJqmOQja6DpxarLj6Jj1U7JuoS8PvL4nEqDaQhy22p4=
+github.com/ultraware/whitespace v0.1.1 h1:bTPOGejYFulW3PkcrqkeQwOd6NKOOXvmGD9bo/Gk8VQ=
+github.com/ultraware/whitespace v0.1.1/go.mod h1:XcP1RLD81eV4BW8UhQlpaR+SDc2givTvyI8a586WjW8=
+github.com/uudashr/gocognit v1.1.3 h1:l+a111VcDbKfynh+airAy/DJQKaXh2m9vkoysMPSZyM=
+github.com/uudashr/gocognit v1.1.3/go.mod h1:aKH8/e8xbTRBwjbCkwZ8qt4l2EpKXl31KMHgSS+lZ2U=
+github.com/vektra/mockery/v2 v2.52.1 h1:ejpWJSsInVNsFUvaAX4szecrpYxErN+Ny5X+0RXBP+s=
+github.com/vektra/mockery/v2 v2.52.1/go.mod h1:ZJeus9igl4Uf8FGLwXZgtCnp2XUDFD9Mkipi7nsObq0=
+github.com/vektra/mockery/v3 v3.7.0/go.mod h1:z9Wr23Ha8etImqQwS3boTNR9WkjX6tIklW5c88DRkSw=
+github.com/xen0n/gosmopolitan v1.2.2 h1:/p2KTnMzwRexIW8GlKawsTWOxn7UHA+jCMF/V8HHtvU=
+github.com/xen0n/gosmopolitan v1.2.2/go.mod h1:7XX7Mj61uLYrj0qmeN0zi7XDon9JRAEhYQqAPLVNTeg=
+github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM=
+github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk=
+github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5JsjqtoFs=
+github.com/yeya24/promlinter v0.3.0/go.mod h1:cDfJQQYv9uYciW60QT0eeHlFodotkYZlL+YcPQN+mW4=
+github.com/ykadowak/zerologlint v0.1.5 h1:Gy/fMz1dFQN9JZTPjv1hxEk+sRWm05row04Yoolgdiw=
+github.com/ykadowak/zerologlint v0.1.5/go.mod h1:KaUskqF3e/v59oPmdq1U1DnKcuHokl2/K1U4pmIELKg=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+gitlab.com/bosi/decorder v0.4.2 h1:qbQaV3zgwnBZ4zPMhGLW4KZe7A7NwxEhJx39R3shffo=
+gitlab.com/bosi/decorder v0.4.2/go.mod h1:muuhHoaJkA9QLcYHq4Mj8FJUwDZ+EirSHRiaTcTf6T8=
+go-simpler.org/assert v0.9.0 h1:PfpmcSvL7yAnWyChSjOz6Sp6m9j5lyK8Ok9pEL31YkQ=
+go-simpler.org/assert v0.9.0/go.mod h1:74Eqh5eI6vCK6Y5l3PI8ZYFXG4Sa+tkr70OIPJAUr28=
+go-simpler.org/musttag v0.12.2 h1:J7lRc2ysXOq7eM8rwaTYnNrHd5JwjppzB6mScysB2Cs=
+go-simpler.org/musttag v0.12.2/go.mod h1:uN1DVIasMTQKk6XSik7yrJoEysGtR2GRqvWnI9S7TYM=
+go-simpler.org/sloglint v0.7.2 h1:Wc9Em/Zeuu7JYpl+oKoYOsQSy2X560aVueCW/m6IijY=
+go-simpler.org/sloglint v0.7.2/go.mod h1:US+9C80ppl7VsThQclkM7BkCHQAzuz8kHLsW3ppuluo=
+go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
+go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
+go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw=
+go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8=
+go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4=
+go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.19.0 h1:Nw7Dv4lwvGrI68+wULbcq7su9K2cebeCUrDjVrUJHxM=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.19.0/go.mod h1:1MsF6Y7gTqosgoZvHlzcaaM8DIMNZgJh87ykokoNH7Y=
+go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc=
+go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8=
+go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE=
+go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg=
+go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4=
+go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ=
+go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8=
+go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
+go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
+go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
+go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
+golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk=
+golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ=
+golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
+golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
+golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
+golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f h1:phY1HzDcf18Aq9A8KkmRtY9WvOFIxN8wgfvy6Zm1DV8=
+golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
+golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
+golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
+golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
+golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
+golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=
+golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
+golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211105183446-c75c47738b0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
+golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
+golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
+golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
+golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
+golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190321232350-e250d351ecad/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200329025819-fd4102a86c65/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200820010801-b793a1359eac/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20201023174141-c8cfbd0f21e6/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.1-0.20210205202024-ef80cdb6ec6d/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU=
+golang.org/x/tools v0.1.1-0.20210302220138-2ac05c832e1a/go.mod h1:9bzcO0MWcOuT0tm1iBGzDVPshzfwoVvREIui8C+MHqU=
+golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
+golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
+golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
+golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
+golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
+gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
+google.golang.org/api v0.196.0 h1:k/RafYqebaIJBO3+SMnfEGtFVlvp5vSgqTUF54UN/zg=
+google.golang.org/api v0.196.0/go.mod h1:g9IL21uGkYgvQ5BZg6BAtoGJQIm8r6EgaAbpNey5wBE=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20181107211654-5fc9ac540362/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 h1:BulPr26Jqjnd4eYDVe+YvyR7Yc2vJGkO5/0UxD0/jZU=
+google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:hL97c3SYopEHblzpxRL4lSs523++l8DYxGM1FQiYmb4=
+google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed h1:3RgNmBoI9MZhsj3QxC+AP/qQhNwpCLOvYDYYsFrhFt0=
+google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:OCdP9MfskevB/rbYvHTsXTtKC+3bHWajPdoKgjcYkfo=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c=
+google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.5.1 h1:4bH5o3b5ZULQ4UrBmP+63W9r7qIkqJClEA9ko5YKx+I=
+honnef.co/go/tools v0.5.1/go.mod h1:e9irvo83WDG9/irijV44wr3tbhcFeRnfpVlRqVwpzMs=
+k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw=
+k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg=
+k8s.io/apiextensions-apiserver v0.28.0 h1:CszgmBL8CizEnj4sj7/PtLGey6Na3YgWyGCPONv7E9E=
+k8s.io/apiextensions-apiserver v0.28.0/go.mod h1:uRdYiwIuu0SyqJKriKmqEN2jThIJPhVmOWETm8ud1VE=
+k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ=
+k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU=
+k8s.io/client-go v0.28.1 h1:pRhMzB8HyLfVwpngWKE8hDcXRqifh1ga2Z/PU9SXVK8=
+k8s.io/client-go v0.28.1/go.mod h1:pEZA3FqOsVkCc07pFVzK076R+P/eXqsgx5zuuRWukNE=
+k8s.io/component-base v0.28.1 h1:LA4AujMlK2mr0tZbQDZkjWbdhTV5bRyEyAFe0TJxlWg=
+k8s.io/component-base v0.28.1/go.mod h1:jI11OyhbX21Qtbav7JkhehyBsIRfnO8oEgoAR12ArIU=
+k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
+k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ=
+k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
+k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk=
+k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+mvdan.cc/gofumpt v0.7.0 h1:bg91ttqXmi9y2xawvkuMXyvAA/1ZGJqYAEGjXuP0JXU=
+mvdan.cc/gofumpt v0.7.0/go.mod h1:txVFJy/Sc/mvaycET54pV8SW8gWxTlUuGHVEcncmNUo=
+mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f h1:lMpcwN6GxNbWtbpI1+xzFLSW8XzX0u72NttUGVFjO3U=
+mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f/go.mod h1:RSLa7mKKCNeTTMHBw5Hsy2rfJmd6O2ivt9Dw9ZqCQpQ=
+sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU=
+sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
diff --git a/boilerplate/flyte/golang_support_tools/tools.go b/boilerplate/flyte/golang_support_tools/tools.go
new file mode 100644
index 00000000000..fa58d223529
--- /dev/null
+++ b/boilerplate/flyte/golang_support_tools/tools.go
@@ -0,0 +1,13 @@
+//go:build tools
+// +build tools
+
+package tools
+
+import (
+ _ "github.com/vektra/mockery/v3/cmd"
+ _ "github.com/alvaroloes/enumer"
+ _ "github.com/golangci/golangci-lint/cmd/golangci-lint"
+ _ "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc"
+
+ _ "github.com/flyteorg/flyte/flytestdlib/cli/pflags"
+)
diff --git a/boilerplate/flyte/golang_test_targets/Makefile b/boilerplate/flyte/golang_test_targets/Makefile
new file mode 100644
index 00000000000..64920149177
--- /dev/null
+++ b/boilerplate/flyte/golang_test_targets/Makefile
@@ -0,0 +1,59 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+LINT_FLAGS ?=
+
+.PHONY: download_tooling
+download_tooling: #download dependencies (including test deps) for the package
+ @../boilerplate/flyte/golang_test_targets/download_tooling.sh
+
+.PHONY: generate
+generate: download_tooling #generate go code
+ @../boilerplate/flyte/golang_test_targets/go-gen.sh
+
+.PHONY: lint
+lint: download_tooling #lints the package for common code smells
+ GL_DEBUG=linters_output,env golangci-lint run $(LINT_FLAGS) --timeout=5m --exclude deprecated -v
+
+.PHONY: lint-fix
+lint-fix: LINT_FLAGS=--fix
+lint-fix: lint
+
+.PHONY: mod_download
+mod_download: #download dependencies (including test deps) for the package
+ go mod download
+
+.PHONY: install
+install: download_tooling mod_download
+
+.PHONY: show
+show:
+ go list -m all
+
+.PHONY: test_unit
+test_unit:
+ go test -cover ./... -race
+
+.PHONY: test_benchmark
+test_benchmark:
+ go test -bench . ./...
+
+.PHONY: test_unit_cover
+test_unit_cover:
+ go test ./... -coverprofile /tmp/cover.out -covermode=count
+ go tool cover -func /tmp/cover.out
+
+.PHONY: test_unit_visual
+test_unit_visual:
+ go test ./... -coverprofile /tmp/cover.out -covermode=count
+ go tool cover -html=/tmp/cover.out
+
+.PHONY: test_unit_codecov
+test_unit_codecov:
+ go test ./... -race -coverprofile=coverage.txt -covermode=atomic
+
+.PHONY: go-tidy
+go-tidy:
+ go mod tidy
diff --git a/boilerplate/flyte/golang_test_targets/Readme.rst b/boilerplate/flyte/golang_test_targets/Readme.rst
new file mode 100644
index 00000000000..700feb33a2f
--- /dev/null
+++ b/boilerplate/flyte/golang_test_targets/Readme.rst
@@ -0,0 +1,33 @@
+Golang Test Targets
+~~~~~~~~~~~~~~~~~~~
+
+Provides an ``install`` make target that uses ``go mod`` to install golang dependencies.
+
+Provides a ``lint`` make target that uses golangci to lint your code.
+
+Provides a ``lint-fix`` make target that uses golangci to lint and fix your code in place.
+
+Provides a ``test_unit`` target for unit tests.
+
+Provides a ``test_unit_cover`` target for analysing coverage of unit tests, which will output the coverage of each function and total statement coverage.
+
+Provides a ``test_unit_visual`` target for visualizing coverage of unit tests through an interactive html code heat map.
+
+Provides a ``test_benchmark`` target for benchmark tests.
+
+**To Enable:**
+
+Add ``flyteorg/golang_test_targets`` to your ``boilerplate/update.cfg`` file.
+
+Make sure you're using ``go mod`` for dependency management.
+
+Provide a ``.golangci`` configuration (the lint target requires it).
+
+Add ``include boilerplate/flyte/golang_test_targets/Makefile`` in your main ``Makefile`` _after_ your REPOSITORY environment variable
+
+::
+
+ REPOSITORY=
+ include boilerplate/flyte/golang_test_targets/Makefile
+
+(this ensures the extra make targets get included in your main Makefile)
diff --git a/boilerplate/flyte/golang_test_targets/download_tooling.sh b/boilerplate/flyte/golang_test_targets/download_tooling.sh
new file mode 100755
index 00000000000..a6bc77245fd
--- /dev/null
+++ b/boilerplate/flyte/golang_test_targets/download_tooling.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Everything in this file needs to be installed outside of current module
+# The reason we cannot turn off module entirely and install is that we need the replace statement in go.mod
+# because we are installing a mockery fork. Turning it off would result installing the original not the fork.
+# We also want to version all the other tools. We also want to be able to run go mod tidy without removing the version
+# pins. To facilitate this, we're maintaining two sets of go.mod/sum files - the second one only for tooling. This is
+# the same approach that go 1.14 will take as well.
+# See:
+# https://github.com/flyteorg/flyte/issues/129
+# https://github.com/golang/go/issues/30515 for some background context
+# https://github.com/go-modules-by-example/index/blob/5ec250b4b78114a55001bd7c9cb88f6e07270ea5/010_tools/README.md
+
+set -e
+
+# List of tools to go get
+# In the format of ":" or ":" if no cli
+tools=(
+ "github.com/vektra/mockery/v3@v3.7.0"
+ "github.com/golangci/golangci-lint/cmd/golangci-lint"
+ "github.com/daixiang0/gci"
+ "github.com/alvaroloes/enumer"
+ "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc"
+)
+
+# This ensures pflags are up to date.
+make -C "${REPO_ROOT}/flytestdlib" compile
+GO_BIN="$(go env GOPATH)/bin"
+mkdir -p "${GO_BIN}"
+cp "${REPO_ROOT}/flytestdlib/bin/pflags" "${GO_BIN}"
+
+tmp_dir="$(mktemp -d -t gotooling-XXX)"
+echo "Using temp directory ${tmp_dir}"
+cp -R ../boilerplate/flyte/golang_support_tools/* "${tmp_dir}"
+pushd "${tmp_dir}"
+
+for tool in "${tools[@]}"; do
+ echo "Installing ${tool}"
+ GO111MODULE=on go install "${tool}"
+done
+
+popd
diff --git a/boilerplate/flyte/golang_test_targets/go-gen.sh b/boilerplate/flyte/golang_test_targets/go-gen.sh
new file mode 100755
index 00000000000..b954df0b147
--- /dev/null
+++ b/boilerplate/flyte/golang_test_targets/go-gen.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -ex
+
+echo "Running go generate"
+go generate ./...
+go mod tidy
+# This section is used by GitHub workflow to ensure that the generation step was run
+if [ -n "$DELTA_CHECK" ]; then
+ DIRTY=$(git status --porcelain)
+ if [ -n "$DIRTY" ]; then
+ echo "FAILED: Go code updated without committing generated code."
+ echo "Ensure make generate has run and all changes are committed."
+ DIFF=$(git diff)
+ echo "diff detected: $DIFF"
+ DIFF=$(git diff --name-only)
+ echo "files different: $DIFF"
+ exit 1
+ else
+ echo "SUCCESS: Generated code is up to date."
+ fi
+fi
diff --git a/boilerplate/flyte/golangci_file/.golangci.yml b/boilerplate/flyte/golangci_file/.golangci.yml
new file mode 100644
index 00000000000..7f4dbc80e8f
--- /dev/null
+++ b/boilerplate/flyte/golangci_file/.golangci.yml
@@ -0,0 +1,40 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+run:
+ skip-dirs:
+ - pkg/client
+
+linters:
+ disable-all: true
+ enable:
+ - deadcode
+ - errcheck
+ - gas
+ - gci
+ - goconst
+ - goimports
+ - golint
+ - gosimple
+ - govet
+ - ineffassign
+ - misspell
+ - nakedret
+ - staticcheck
+ - structcheck
+ - typecheck
+ - unconvert
+ - unparam
+ - unused
+ - varcheck
+
+linters-settings:
+ gci:
+ custom-order: true
+ sections:
+ - standard
+ - default
+ - prefix(github.com/flyteorg)
+ skip-generated: true
diff --git a/boilerplate/flyte/golangci_file/Readme.rst b/boilerplate/flyte/golangci_file/Readme.rst
new file mode 100644
index 00000000000..e4cbd18b969
--- /dev/null
+++ b/boilerplate/flyte/golangci_file/Readme.rst
@@ -0,0 +1,8 @@
+GolangCI File
+~~~~~~~~~~~~~
+
+Provides a ``.golangci`` file with the linters we've agreed upon.
+
+**To Enable:**
+
+Add ``flyteorg/golangci_file`` to your ``boilerplate/update.cfg`` file.
diff --git a/boilerplate/flyte/golangci_file/update.sh b/boilerplate/flyte/golangci_file/update.sh
new file mode 100755
index 00000000000..ab2f85c680d
--- /dev/null
+++ b/boilerplate/flyte/golangci_file/update.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+# Clone the .golangci file
+echo " - copying ${DIR}/.golangci to the root directory."
+cp ${DIR}/.golangci.yml ${DIR}/../../../.golangci.yml
diff --git a/boilerplate/flyte/precommit/Makefile b/boilerplate/flyte/precommit/Makefile
new file mode 100644
index 00000000000..3c6f17d6b26
--- /dev/null
+++ b/boilerplate/flyte/precommit/Makefile
@@ -0,0 +1,9 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+
+.PHONY: setup-precommit
+setup-precommit: #setup the precommit
+ @boilerplate/flyte/precommit/update.sh
diff --git a/boilerplate/flyte/precommit/hooks/pre-push b/boilerplate/flyte/precommit/hooks/pre-push
new file mode 100755
index 00000000000..f161cfe8565
--- /dev/null
+++ b/boilerplate/flyte/precommit/hooks/pre-push
@@ -0,0 +1,41 @@
+DUMMY_SHA=0000000000000000000000000000000000000000
+
+echo "Running pre-push check; to skip this step use 'push --no-verify'"
+
+while read LOCAL_REF LOCAL_SHA REMOTE_REF REMOTE_SHA
+do
+ if [ "$LOCAL_SHA" = $DUMMY_SHA ]
+ then
+ # Branch deleted. Do nothing.
+ exit 0
+ else
+ if [ "$REMOTE_SHA" = $DUMMY_SHA ]
+ then
+ # New branch. Verify the last commit, since this is very likely where the new code is
+ # (though there is no way to know for sure). In the extremely uncommon case in which someone
+ # pushes more than 1 new commit to a branch, CI will enforce full checking.
+ RANGE="$LOCAL_SHA~1..$LOCAL_SHA"
+ else
+ # Updating branch. Verify new commits.
+ RANGE="$REMOTE_SHA..$LOCAL_SHA"
+ fi
+
+ # Verify DCO signoff. We do this before the format checker, since it has
+ # some probability of failing spuriously, while this check never should.
+ #
+ # In general, we can't assume that the commits are signed off by author
+ # pushing, so we settle for just checking that there is a signoff at all.
+ SIGNED_OFF=$(git rev-list --no-merges --grep "^Signed-off-by: " "$RANGE")
+ NOT_SIGNED_OFF=$(git rev-list --no-merges "$RANGE" | grep -Fxv "$SIGNED_OFF")
+ if [ -n "$NOT_SIGNED_OFF" ]
+ then
+ echo >&2 "ERROR: The following commits do not have DCO signoff:"
+ while read -r commit; do
+ echo " $(git log --pretty=oneline --abbrev-commit -n 1 $commit)"
+ done <<< "$NOT_SIGNED_OFF"
+ exit 1
+ fi
+ fi
+done
+
+exit 0
diff --git a/boilerplate/flyte/precommit/hooks/prepare-commit-msg b/boilerplate/flyte/precommit/hooks/prepare-commit-msg
new file mode 100755
index 00000000000..8148d104b84
--- /dev/null
+++ b/boilerplate/flyte/precommit/hooks/prepare-commit-msg
@@ -0,0 +1,16 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+# $ ln -s ../../support/hooks/prepare-commit-msg .git/hooks/prepare-commit-msg
+
+COMMIT_MESSAGE_FILE="$1"
+AUTHOR=$(git var GIT_AUTHOR_IDENT)
+SIGNOFF=$(echo $AUTHOR | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p')
+
+# Check for DCO signoff message. If one doesn't exist, append one and then warn
+# the user that you did so.
+if ! $(grep -qs "^$SIGNOFF" "$COMMIT_MESSAGE_FILE") ; then
+ echo "\n$SIGNOFF" >> "$COMMIT_MESSAGE_FILE"
+ echo "Appended the following signoff to the end of the commit message:\n $SIGNOFF\n"
+fi
diff --git a/boilerplate/flyte/precommit/update.sh b/boilerplate/flyte/precommit/update.sh
new file mode 100755
index 00000000000..971c8386c1f
--- /dev/null
+++ b/boilerplate/flyte/precommit/update.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+set -e
+
+# Helper script for Automatically add DCO signoff with commit hooks
+# Taken from Envoy https://gitlab.cncf.ci/envoyproxy/envoy
+if [ ! "$PWD" == "$(git rev-parse --show-toplevel)" ]; then
+ cat >&2 <<__EOF__
+ERROR: this script must be run at the root of the envoy source tree
+__EOF__
+ exit 1
+fi
+
+# Helper functions that calculate `abspath` and `relpath`. Taken from Mesos
+# commit 82b040a60561cf94dec3197ea88ae15e57bcaa97, which also carries the Apache
+# V2 license, and has deployed this code successfully for some time.
+abspath() {
+ cd "$(dirname "${1}")"
+ echo "${PWD}"/"$(basename "${1}")"
+ cd "${OLDPWD}"
+}
+relpath() {
+ local FROM TO UP
+ FROM="$(abspath "${1%/}")" TO="$(abspath "${2%/}"/)"
+ while test "${TO}" = "${TO#"${FROM}"/}" \
+ -a "${TO}" != "${FROM}"; do
+ FROM="${FROM%/*}" UP="../${UP}"
+ done
+ TO="${UP%/}${TO#${FROM}}"
+ echo "${TO:-.}"
+}
+
+# Try to find the `.git` directory, even if it's not in Flyte project root (as
+# it wouldn't be if, say, this were in a submodule). The "blessed" but fairly
+# new way to do this is to use `--git-common-dir`.
+DOT_GIT_DIR=$(git rev-parse --git-common-dir)
+if test ! -d "${DOT_GIT_DIR}"; then
+ # If `--git-common-dir` is not available, fall back to older way of doing it.
+ DOT_GIT_DIR=$(git rev-parse --git-dir)
+fi
+
+mkdir -p ${DOT_GIT_DIR}/hooks
+
+HOOKS_DIR="${DOT_GIT_DIR}/hooks"
+HOOKS_DIR_RELPATH=$(relpath "${HOOKS_DIR}" "${PWD}")
+
+if [ ! -e "${HOOKS_DIR}/prepare-commit-msg" ]; then
+ echo "Installing hook 'prepare-commit-msg'"
+ ln -s "${HOOKS_DIR_RELPATH}/boilerplate/flyte/precommit/hooks/prepare-commit-msg" "${HOOKS_DIR}/prepare-commit-msg"
+fi
+
+if [ ! -e "${HOOKS_DIR}/pre-push" ]; then
+ echo "Installing hook 'pre-push'"
+ ln -s "${HOOKS_DIR_RELPATH}/boilerplate/flyte/precommit/hooks/pre-push" "${HOOKS_DIR}/pre-push"
+fi
diff --git a/boilerplate/flyte/pull_request_template/Readme.rst b/boilerplate/flyte/pull_request_template/Readme.rst
new file mode 100644
index 00000000000..ee54437252e
--- /dev/null
+++ b/boilerplate/flyte/pull_request_template/Readme.rst
@@ -0,0 +1,8 @@
+Pull Request Template
+~~~~~~~~~~~~~~~~~~~~~
+
+Provides a Pull Request template.
+
+**To Enable:**
+
+Add ``flyteorg/golang_test_targets`` to your ``boilerplate/update.cfg`` file.
diff --git a/boilerplate/flyte/pull_request_template/pull_request_template.md b/boilerplate/flyte/pull_request_template/pull_request_template.md
new file mode 100644
index 00000000000..9cdab99b465
--- /dev/null
+++ b/boilerplate/flyte/pull_request_template/pull_request_template.md
@@ -0,0 +1,35 @@
+## _Read then delete this section_
+
+_- Make sure to use a concise title for the pull-request._
+
+_- Use #patch, #minor or #major in the pull-request title to bump the corresponding version. Otherwise, the patch version
+will be bumped. [More details](https://github.com/marketplace/actions/github-tag-bump)_
+
+# TL;DR
+_Please replace this text with a description of what this PR accomplishes._
+
+## Type
+ - [ ] Bug Fix
+ - [ ] Feature
+ - [ ] Plugin
+
+## Are all requirements met?
+
+ - [ ] Code completed
+ - [ ] Smoke tested
+ - [ ] Unit tests added
+ - [ ] Code documentation added
+ - [ ] Any pending items have an associated Issue
+
+## Complete description
+ _How did you fix the bug, make the feature etc. Link to any design docs etc_
+
+## Tracking Issue
+_Remove the '*fixes*' keyword if there will be multiple PRs to fix the linked issue_
+
+fixes https://github.com/flyteorg/flyte/issues/
+
+## Follow-up issue
+_NA_
+OR
+_https://github.com/flyteorg/flyte/issues/_
diff --git a/boilerplate/flyte/pull_request_template/update.sh b/boilerplate/flyte/pull_request_template/update.sh
new file mode 100755
index 00000000000..051e9dbce0e
--- /dev/null
+++ b/boilerplate/flyte/pull_request_template/update.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+cp ${DIR}/pull_request_template.md ${DIR}/../../../pull_request_template.md
diff --git a/boilerplate/update.sh b/boilerplate/update.sh
new file mode 100755
index 00000000000..73de4dc91c3
--- /dev/null
+++ b/boilerplate/update.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+OUT="$(mktemp -d)"
+trap 'rm -fr $OUT' EXIT
+
+git clone https://github.com/flyteorg/boilerplate.git "${OUT}"
+
+echo "Updating the update.sh script."
+cp "${OUT}/boilerplate/update.sh" "${DIR}/update.sh"
+
+CONFIG_FILE="${DIR}/update.cfg"
+README="https://github.com/flyteorg/boilerplate/blob/master/Readme.rst"
+
+if [ ! -f "$CONFIG_FILE" ]; then
+ echo "$CONFIG_FILE not found."
+ echo "This file is required in order to select which features to include."
+ echo "See $README for more details."
+ exit 1
+fi
+
+if [ -z "$REPOSITORY" ]; then
+ echo "$REPOSITORY is required to run this script"
+ echo "See $README for more details."
+ exit 1
+fi
+
+while read -r directory junk; do
+ # Skip comment lines (which can have leading whitespace)
+ if [[ "$directory" == '#'* ]]; then
+ continue
+ fi
+ # Skip blank or whitespace-only lines
+ if [[ "$directory" == "" ]]; then
+ continue
+ fi
+ # Lines like
+ # valid/path other_junk
+ # are not acceptable, unless `other_junk` is a comment
+ if [[ "$junk" != "" ]] && [[ "$junk" != '#'* ]]; then
+ echo "Invalid config! Only one directory is allowed per line. Found '$junk'"
+ exit 1
+ fi
+
+ dir_path="${OUT}/boilerplate/${directory}"
+ # Make sure the directory exists
+ if ! [[ -d "$dir_path" ]]; then
+ echo "Invalid boilerplate directory: '$directory'"
+ exit 1
+ fi
+
+ echo "***********************************************************************************"
+ echo "$directory is configured in update.cfg."
+ echo "-----------------------------------------------------------------------------------"
+ echo "syncing files from source."
+ rm -rf "${DIR:?}/${directory}"
+ mkdir -p "$(dirname "${DIR}"/"${directory}")"
+ cp -r "$dir_path" "${DIR}/${directory}"
+ if [ -f "${DIR}/${directory}/update.sh" ]; then
+ echo "executing ${DIR}/${directory}/update.sh"
+ "${DIR}/${directory}/update.sh"
+ fi
+ echo "***********************************************************************************"
+ echo ""
+done < "$CONFIG_FILE"
diff --git a/buf.gen.go.yaml b/buf.gen.go.yaml
new file mode 100644
index 00000000000..fcba205edc6
--- /dev/null
+++ b/buf.gen.go.yaml
@@ -0,0 +1,40 @@
+version: v2
+managed:
+ enabled: true
+ override:
+ - file_option: optimize_for
+ value: CODE_SIZE
+ # Use managed mode. Ref: https://protovalidate.com/quickstart/#depend-on-protovalidate
+ disable:
+ - file_option: go_package
+ - module: buf.build/bufbuild/protovalidate
+plugins:
+ - remote: buf.build/protocolbuffers/go:v1.30.0
+ out: gen/go
+ opt:
+ - paths=source_relative
+ - remote: buf.build/grpc/go:v1.3.0
+ out: gen/go
+ opt:
+ - paths=source_relative
+ - require_unimplemented_servers=false
+ - remote: buf.build/connectrpc/go:v1.16.2
+ out: gen/go
+ opt:
+ - paths=source_relative
+ - remote: buf.build/bufbuild/validate-go:v1.2.1
+ out: gen/go
+ opt:
+ - paths=source_relative
+ - remote: buf.build/grpc-ecosystem/gateway:v2.15.2
+ out: gen/go/gateway
+ opt:
+ - paths=import
+ - module=github.com/flyteorg/flyte/v2/gen/go
+ - standalone=true
+ - allow_delete_body=true
+ - remote: buf.build/grpc-ecosystem/openapiv2:v2.27.4
+ out: gen/go/gateway
+ opt:
+ - json_names_for_fields=false
+ - allow_delete_body=true
diff --git a/buf.gen.python.yaml b/buf.gen.python.yaml
new file mode 100644
index 00000000000..bfaf6c8c50a
--- /dev/null
+++ b/buf.gen.python.yaml
@@ -0,0 +1,19 @@
+version: v2
+managed:
+ enabled: true
+ override:
+ - file_option: optimize_for
+ value: CODE_SIZE
+ # Use managed mode: https://buf.build/docs/generate/managed-mode/#managed-mode
+plugins:
+ - remote: buf.build/protocolbuffers/python:v24.1
+ out: gen/python
+ include_imports: true
+ - remote: buf.build/protocolbuffers/pyi:v24.1
+ out: gen/python
+ include_imports: true
+ - remote: buf.build/grpc/python:v1.58.1
+ out: gen/python
+ include_imports: true
+ - remote: buf.build/connectrpc/python:v0.9.0
+ out: gen/python
diff --git a/buf.gen.rust.yaml b/buf.gen.rust.yaml
new file mode 100644
index 00000000000..f2d2e276165
--- /dev/null
+++ b/buf.gen.rust.yaml
@@ -0,0 +1,17 @@
+version: v2
+managed:
+ enabled: true
+ # Use managed mode: https://buf.build/docs/generate/managed-mode/#managed-mode
+ disable:
+ - module: buf.build/bufbuild/protovalidate
+plugins:
+ - remote: buf.build/community/neoeinstein-prost:v0.4.0
+ out: gen/rust/src
+ opt:
+ - file_descriptor_set
+ - type_attribute=.=#[pyo3::pyclass(dict\, get_all\, set_all)]
+ - compile_well_known_types
+ - remote: buf.build/community/neoeinstein-tonic:v0.4.1
+ out: gen/rust/src
+ opt:
+ - compile_well_known_types
diff --git a/buf.gen.ts.yaml b/buf.gen.ts.yaml
new file mode 100644
index 00000000000..3c9fd2f7d80
--- /dev/null
+++ b/buf.gen.ts.yaml
@@ -0,0 +1,16 @@
+version: v2
+managed:
+ enabled: true
+ override:
+ - file_option: optimize_for
+ value: CODE_SIZE
+ # Use managed mode: https://buf.build/docs/generate/managed-mode/#managed-mode
+ disable:
+ - module: buf.build/bufbuild/protovalidate
+plugins:
+ - remote: buf.build/bufbuild/es:v2.2.5
+ out: gen/ts
+ include_imports: true
+ opt:
+ - target=ts
+ - import_extension=.ts
diff --git a/buf.lock b/buf.lock
new file mode 100644
index 00000000000..553d02e710d
--- /dev/null
+++ b/buf.lock
@@ -0,0 +1,9 @@
+# Generated by buf. DO NOT EDIT.
+version: v2
+deps:
+ - name: buf.build/bufbuild/protovalidate
+ commit: 6c6e0d3c608e4549802254a2eee81bc8
+ digest: b5:a7ca081f38656fc0f5aaa685cc111d3342876723851b47ca6b80cbb810cbb2380f8c444115c495ada58fa1f85eff44e68dc54a445761c195acdb5e8d9af675b6
+ - name: buf.build/googleapis/googleapis
+ commit: 62f35d8aed1149c291d606d958a7ce32
+ digest: b5:d66bf04adc77a0870bdc9328aaf887c7188a36fb02b83a480dc45ef9dc031b4d39fc6e9dc6435120ccf4fe5bfd5c6cb6592533c6c316595571f9a31420ab47fe
diff --git a/buf.yaml b/buf.yaml
new file mode 100755
index 00000000000..26c09a38ad2
--- /dev/null
+++ b/buf.yaml
@@ -0,0 +1,28 @@
+version: v2
+modules:
+ - path: .
+ name: buf.build/flyteorg/flyte
+ excludes:
+ - .claude
+deps:
+ - buf.build/googleapis/googleapis:62f35d8aed1149c291d606d958a7ce32
+ - buf.build/bufbuild/protovalidate:v0.14.1
+lint:
+ except:
+ - PACKAGE_VERSION_SUFFIX
+ - FIELD_LOWER_SNAKE_CASE
+ - ENUM_VALUE_PREFIX
+ - ENUM_ZERO_VALUE_SUFFIX
+ ignore_only:
+ RPC_REQUEST_RESPONSE_UNIQUE:
+ - flyteidl2/cacheservice/cacheservice.proto
+ - flyteidl2/cacheservice/v2/cacheservice.proto
+ - flyteidl2/project/project_service.proto
+ RPC_REQUEST_STANDARD_NAME:
+ - flyteidl2/cacheservice/cacheservice.proto
+ - flyteidl2/cacheservice/v2/cacheservice.proto
+ RPC_RESPONSE_STANDARD_NAME:
+ - flyteidl2/cacheservice/cacheservice.proto
+ - flyteidl2/cacheservice/v2/cacheservice.proto
+ SERVICE_SUFFIX:
+ - flyteidl2/datacatalog/datacatalog.proto
diff --git a/cache_service/cmd/main.go b/cache_service/cmd/main.go
new file mode 100644
index 00000000000..9a53d16aaf7
--- /dev/null
+++ b/cache_service/cmd/main.go
@@ -0,0 +1,47 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/app"
+ "github.com/flyteorg/flyte/v2/cache_service"
+ cacheserviceconfig "github.com/flyteorg/flyte/v2/cache_service/config"
+ "github.com/flyteorg/flyte/v2/flytestdlib/contextutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/database"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils/labeled"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+)
+
+func main() {
+ a := &app.App{
+ Name: "cache-service",
+ Short: "Cache Service for Flyte",
+ Setup: func(ctx context.Context, sc *app.SetupContext) error {
+ cfg := cacheserviceconfig.GetConfig()
+ sc.Host = cfg.Server.Host
+ sc.Port = cfg.Server.Port
+
+ db, err := database.GetDB(ctx, database.GetConfig())
+ if err != nil {
+ return fmt.Errorf("failed to initialize database: %w", err)
+ }
+ sc.DB = db
+
+ labeled.SetMetricKeys(contextutils.ProjectKey, contextutils.DomainKey, contextutils.WorkflowIDKey, contextutils.TaskIDKey)
+ dataStore, err := storage.NewDataStore(storage.GetConfig(), promutils.NewScope("cache-service"))
+ if err != nil {
+ return fmt.Errorf("failed to initialize storage: %w", err)
+ }
+ sc.DataStore = dataStore
+
+ return cache_service.Setup(ctx, sc)
+ },
+ }
+
+ if err := a.Run(); err != nil {
+ os.Exit(1)
+ }
+}
diff --git a/cache_service/config/config.go b/cache_service/config/config.go
new file mode 100644
index 00000000000..5863f699932
--- /dev/null
+++ b/cache_service/config/config.go
@@ -0,0 +1,38 @@
+package config
+
+import (
+ "time"
+
+ stdconfig "github.com/flyteorg/flyte/v2/flytestdlib/config"
+)
+
+const configSectionKey = "cache_service"
+
+//go:generate pflags Config --default-var=defaultConfig
+
+var defaultConfig = &Config{
+ Server: ServerConfig{
+ Port: 8094,
+ Host: "0.0.0.0",
+ },
+ HeartbeatGracePeriodMultiplier: 3,
+ MaxReservationHeartbeat: stdconfig.Duration{Duration: 10 * time.Second},
+}
+
+var configSection = stdconfig.MustRegisterSection(configSectionKey, defaultConfig)
+
+type Config struct {
+ Server ServerConfig `json:"server"`
+
+ HeartbeatGracePeriodMultiplier int `json:"heartbeatGracePeriodMultiplier" pflag:",Number of heartbeats before a reservation expires without an extension."`
+ MaxReservationHeartbeat stdconfig.Duration `json:"maxReservationHeartbeat" pflag:",Maximum reservation heartbeat interval."`
+}
+
+type ServerConfig struct {
+ Port int `json:"port" pflag:",Port to bind the HTTP server"`
+ Host string `json:"host" pflag:",Host to bind the HTTP server"`
+}
+
+func GetConfig() *Config {
+ return configSection.GetConfig().(*Config)
+}
diff --git a/cache_service/manager/main_test.go b/cache_service/manager/main_test.go
new file mode 100644
index 00000000000..61f84ffbb91
--- /dev/null
+++ b/cache_service/manager/main_test.go
@@ -0,0 +1,20 @@
+package manager
+
+import (
+ "context"
+ "os"
+ "testing"
+
+ "github.com/jmoiron/sqlx"
+
+ "github.com/flyteorg/flyte/v2/cache_service/migrations"
+ "github.com/flyteorg/flyte/v2/flytestdlib/database"
+)
+
+var testDB *sqlx.DB
+
+func TestMain(m *testing.M) {
+ os.Exit(database.RunTestMain(m, 15434, "flyte_cache_test", &testDB, func(db *sqlx.DB) error {
+ return migrations.RunMigrations(context.Background(), db)
+ }))
+}
diff --git a/cache_service/manager/manager.go b/cache_service/manager/manager.go
new file mode 100644
index 00000000000..f301b09e5ad
--- /dev/null
+++ b/cache_service/manager/manager.go
@@ -0,0 +1,283 @@
+package manager
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "connectrpc.com/connect"
+ "google.golang.org/protobuf/proto"
+ durationpb "google.golang.org/protobuf/types/known/durationpb"
+ "google.golang.org/protobuf/types/known/timestamppb"
+
+ cacheconfig "github.com/flyteorg/flyte/v2/cache_service/config"
+ repositoryerrors "github.com/flyteorg/flyte/v2/cache_service/repository/errors"
+ "github.com/flyteorg/flyte/v2/cache_service/repository/interfaces"
+ "github.com/flyteorg/flyte/v2/cache_service/repository/models"
+ cacheservicepb "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/cacheservice"
+)
+
+const reservationPrefix = "reservation"
+
+// Manager owns the cache service's core behavior:
+// - read/write cached outputs
+// - maintain cache metadata timestamps
+// - coordinate serialized cache population with reservations
+//
+// The service layer stays thin and only translates V2 transport requests into
+// manager calls. Repository implementations only persist rows. The policy of
+// "who gets to populate cache and when" lives here.
+type Manager struct {
+ outputs interfaces.CachedOutputRepo
+ reservations interfaces.ReservationRepo
+ heartbeatGracePeriodMultiplier int
+ maxReservationHeartbeatInterval time.Duration
+}
+
+type CacheEntry struct {
+ OutputURI string
+ Metadata *cacheservicepb.Metadata
+}
+
+func New(cfg *cacheconfig.Config, outputs interfaces.CachedOutputRepo, reservations interfaces.ReservationRepo) *Manager {
+ maxHeartbeat := cfg.MaxReservationHeartbeat.Duration
+ if maxHeartbeat <= 0 {
+ maxHeartbeat = 10 * time.Second
+ }
+
+ graceMultiplier := cfg.HeartbeatGracePeriodMultiplier
+ if graceMultiplier <= 0 {
+ graceMultiplier = 3
+ }
+
+ return &Manager{
+ outputs: outputs,
+ reservations: reservations,
+ heartbeatGracePeriodMultiplier: graceMultiplier,
+ maxReservationHeartbeatInterval: maxHeartbeat,
+ }
+}
+
+// Get returns the materialized cache entry for the given key, if one exists.
+func (m *Manager) Get(ctx context.Context, request *cacheservicepb.GetCacheRequest) (*CacheEntry, error) {
+ if request.GetKey() == "" {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("key is required"))
+ }
+
+ output, err := m.outputs.Get(ctx, request.GetKey())
+ if err != nil {
+ if repositoryerrors.IsNotFound(err) {
+ return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("cache entry %q not found", request.GetKey()))
+ }
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+
+ metadata, err := unmarshalMetadata(output.Metadata)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+
+ return &CacheEntry{
+ OutputURI: output.OutputURI,
+ Metadata: metadata,
+ }, nil
+}
+
+// Put stores or overwrites the materialized cache entry for a key.
+//
+// In OSS V2 we only persist output URIs. The actual output payload continues to
+// live in object storage; cache service stores the lookup record plus metadata.
+func (m *Manager) Put(ctx context.Context, request *cacheservicepb.PutCacheRequest) error {
+ if request.GetKey() == "" {
+ return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("key is required"))
+ }
+ if request.GetOutput() == nil || request.GetOutput().GetOutputUri() == "" {
+ return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("output_uri is required"))
+ }
+
+ now := time.Now().UTC()
+ existing, err := m.outputs.Get(ctx, request.GetKey())
+ if err != nil && !repositoryerrors.IsNotFound(err) {
+ return connect.NewError(connect.CodeInternal, err)
+ }
+
+ if err == nil {
+ expired := false
+ if maxAge := request.GetOverwrite().GetMaxAge(); maxAge != nil && !existing.LastUpdated.IsZero() {
+ expired = time.Since(existing.LastUpdated) > maxAge.AsDuration()
+ }
+ if !request.GetOverwrite().GetOverwrite() && !expired {
+ return connect.NewError(connect.CodeAlreadyExists, fmt.Errorf("cache entry %q already exists", request.GetKey()))
+ }
+ }
+
+ metadata := mergeMetadata(existing, request.GetOutput().GetMetadata(), now)
+ metadataBytes, err := proto.Marshal(metadata)
+ if err != nil {
+ return connect.NewError(connect.CodeInternal, err)
+ }
+
+ model := &models.CachedOutput{
+ Key: request.GetKey(),
+ OutputURI: request.GetOutput().GetOutputUri(),
+ Metadata: metadataBytes,
+ LastUpdated: metadata.GetLastUpdatedAt().AsTime(),
+ }
+ if err := m.outputs.Put(ctx, model); err != nil {
+ return connect.NewError(connect.CodeInternal, err)
+ }
+
+ return nil
+}
+
+// Delete removes the cache metadata row for a key.
+//
+// This does not delete the referenced object-storage blob. The caller may be
+// pointing at a shared URI, and blob lifecycle is better handled separately.
+func (m *Manager) Delete(ctx context.Context, request *cacheservicepb.DeleteCacheRequest) error {
+ if request.GetKey() == "" {
+ return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("key is required"))
+ }
+ if err := m.outputs.Delete(ctx, request.GetKey()); err != nil {
+ if repositoryerrors.IsNotFound(err) {
+ return connect.NewError(connect.CodeNotFound, fmt.Errorf("cache entry %q not found", request.GetKey()))
+ }
+ return connect.NewError(connect.CodeInternal, err)
+ }
+ return nil
+}
+
+// GetOrExtendReservation returns the active reservation for a cache key,
+// creating or refreshing it when the caller is allowed to own it.
+//
+// This is the coordination path for serialized cache population. On a cache
+// miss, only the active owner should execute and publish the result; other
+// callers observe the current reservation and wait for the cache entry to
+// appear.
+func (m *Manager) GetOrExtendReservation(ctx context.Context, request *cacheservicepb.GetOrExtendReservationRequest, now time.Time) (*cacheservicepb.Reservation, error) {
+ if request.GetKey() == "" {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("key is required"))
+ }
+ if request.GetOwnerId() == "" {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("owner_id is required"))
+ }
+
+ reservationKey := fmt.Sprintf("%s:%s", reservationPrefix, request.GetKey())
+ heartbeat := m.resolvedHeartbeat(request.GetHeartbeatInterval())
+ reservation := &models.Reservation{
+ Key: reservationKey,
+ OwnerID: request.GetOwnerId(),
+ HeartbeatSeconds: int64(heartbeat.Seconds()),
+ ExpiresAt: now.Add(heartbeat * time.Duration(m.heartbeatGracePeriodMultiplier)),
+ }
+
+ _, err := m.reservations.Get(ctx, reservationKey)
+ if err != nil && !repositoryerrors.IsNotFound(err) {
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+
+ // Existing reservations follow one of two paths:
+ // - same owner or expired: current caller can refresh it
+ // - different active owner: keep returning the current holder
+ if err == nil {
+ if err := m.reservations.UpdateIfExpiredOrOwned(ctx, reservation, now); err != nil {
+ if repositoryerrors.IsReservationNotClaimable(err) {
+ // Another caller still owns the reservation or claimed it before we
+ // could refresh an expired one. Re-read to return the current holder.
+ current, getErr := m.reservations.Get(ctx, reservationKey)
+ if getErr != nil {
+ return nil, connect.NewError(connect.CodeInternal, getErr)
+ }
+ return reservationFromModel(current), nil
+ }
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+ return reservationFromModel(reservation), nil
+ }
+
+ if err := m.reservations.Create(ctx, reservation); err != nil {
+ if repositoryerrors.IsAlreadyExists(err) {
+ // Another caller created the reservation after our initial read.
+ current, getErr := m.reservations.Get(ctx, reservationKey)
+ if getErr != nil {
+ return nil, connect.NewError(connect.CodeInternal, getErr)
+ }
+ return reservationFromModel(current), nil
+ }
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+
+ return reservationFromModel(reservation), nil
+}
+
+// ReleaseReservation releases ownership for serialized cache population.
+//
+// Missing reservations are treated as already-released so callers can clean up
+// idempotently.
+func (m *Manager) ReleaseReservation(ctx context.Context, request *cacheservicepb.ReleaseReservationRequest) error {
+ if request.GetKey() == "" {
+ return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("key is required"))
+ }
+ if request.GetOwnerId() == "" {
+ return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("owner_id is required"))
+ }
+
+ reservationKey := fmt.Sprintf("%s:%s", reservationPrefix, request.GetKey())
+ if err := m.reservations.DeleteByKeyAndOwner(ctx, reservationKey, request.GetOwnerId()); err != nil {
+ if repositoryerrors.IsNotFound(err) {
+ return nil
+ }
+ return connect.NewError(connect.CodeInternal, err)
+ }
+ return nil
+}
+
+func (m *Manager) resolvedHeartbeat(requested *durationpb.Duration) time.Duration {
+ heartbeat := m.maxReservationHeartbeatInterval
+ if requested != nil && requested.AsDuration() > 0 && requested.AsDuration() < heartbeat {
+ heartbeat = requested.AsDuration()
+ }
+ return heartbeat
+}
+
+func mergeMetadata(existing *models.CachedOutput, request *cacheservicepb.Metadata, now time.Time) *cacheservicepb.Metadata {
+ var metadata *cacheservicepb.Metadata
+ if request != nil {
+ metadata = proto.Clone(request).(*cacheservicepb.Metadata)
+ } else {
+ metadata = &cacheservicepb.Metadata{}
+ }
+
+ if existing != nil && len(existing.Metadata) > 0 {
+ if existingMetadata, err := unmarshalMetadata(existing.Metadata); err == nil && metadata.GetCreatedAt() == nil {
+ // Keep the original CreatedAt
+ metadata.CreatedAt = existingMetadata.GetCreatedAt()
+ }
+ }
+
+ if metadata.GetCreatedAt() == nil {
+ metadata.CreatedAt = timestamppb.New(now)
+ }
+ metadata.LastUpdatedAt = timestamppb.New(now)
+ return metadata
+}
+
+func unmarshalMetadata(data []byte) (*cacheservicepb.Metadata, error) {
+ metadata := &cacheservicepb.Metadata{}
+ if len(data) == 0 {
+ return metadata, nil
+ }
+ if err := proto.Unmarshal(data, metadata); err != nil {
+ return nil, fmt.Errorf("unmarshal metadata: %w", err)
+ }
+ return metadata, nil
+}
+
+func reservationFromModel(model *models.Reservation) *cacheservicepb.Reservation {
+ return &cacheservicepb.Reservation{
+ Key: model.Key,
+ OwnerId: model.OwnerID,
+ HeartbeatInterval: durationpb.New(time.Duration(model.HeartbeatSeconds) * time.Second),
+ ExpiresAt: timestamppb.New(model.ExpiresAt),
+ }
+}
diff --git a/cache_service/manager/manager_test.go b/cache_service/manager/manager_test.go
new file mode 100644
index 00000000000..77cdbc40eb9
--- /dev/null
+++ b/cache_service/manager/manager_test.go
@@ -0,0 +1,69 @@
+package manager
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ durationpb "google.golang.org/protobuf/types/known/durationpb"
+
+ cacheconfig "github.com/flyteorg/flyte/v2/cache_service/config"
+ "github.com/flyteorg/flyte/v2/cache_service/repository"
+ cacheservicepb "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/cacheservice"
+)
+
+func newTestManager(t *testing.T) *Manager {
+ t.Helper()
+ t.Cleanup(func() {
+ testDB.Exec("DELETE FROM cached_outputs")
+ testDB.Exec("DELETE FROM reservations")
+ })
+
+ cfg := &cacheconfig.Config{
+ HeartbeatGracePeriodMultiplier: 3,
+ }
+ cfg.MaxReservationHeartbeat.Duration = 10 * time.Second
+
+ repos := repository.NewRepository(testDB)
+ return New(cfg, repos.CachedOutputRepo(), repos.ReservationRepo())
+}
+
+func TestManagerPutGetURIOutput(t *testing.T) {
+ m := newTestManager(t)
+ ctx := context.Background()
+
+ err := m.Put(ctx, &cacheservicepb.PutCacheRequest{
+ Key: "k1",
+ Output: &cacheservicepb.CachedOutput{
+ Output: &cacheservicepb.CachedOutput_OutputUri{OutputUri: "s3://bucket/path"},
+ },
+ })
+ require.NoError(t, err)
+
+ entry, err := m.Get(ctx, &cacheservicepb.GetCacheRequest{Key: "k1"})
+ require.NoError(t, err)
+ require.Equal(t, "s3://bucket/path", entry.OutputURI)
+}
+
+func TestManagerReservationReturnsExistingOwnerWhenHeld(t *testing.T) {
+ m := newTestManager(t)
+ ctx := context.Background()
+ now := time.Now().UTC()
+
+ first, err := m.GetOrExtendReservation(ctx, &cacheservicepb.GetOrExtendReservationRequest{
+ Key: "k1",
+ OwnerId: "owner-a",
+ HeartbeatInterval: durationpb.New(2 * time.Second),
+ }, now)
+ require.NoError(t, err)
+ require.Equal(t, "owner-a", first.GetOwnerId())
+
+ second, err := m.GetOrExtendReservation(ctx, &cacheservicepb.GetOrExtendReservationRequest{
+ Key: "k1",
+ OwnerId: "owner-b",
+ HeartbeatInterval: durationpb.New(2 * time.Second),
+ }, now.Add(1*time.Second))
+ require.NoError(t, err)
+ require.Equal(t, "owner-a", second.GetOwnerId())
+}
diff --git a/cache_service/migrations/migrations.go b/cache_service/migrations/migrations.go
new file mode 100644
index 00000000000..2587d441cd3
--- /dev/null
+++ b/cache_service/migrations/migrations.go
@@ -0,0 +1,18 @@
+package migrations
+
+import (
+ "context"
+ "embed"
+
+ "github.com/jmoiron/sqlx"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/database"
+)
+
+//go:embed sql/*.sql
+var migrationFS embed.FS
+
+// RunMigrations applies all pending cache service migrations.
+func RunMigrations(ctx context.Context, db *sqlx.DB) error {
+ return database.Migrate(ctx, db, "cache_service", migrationFS)
+}
diff --git a/cache_service/migrations/sql/20260408110000_init_schema.sql b/cache_service/migrations/sql/20260408110000_init_schema.sql
new file mode 100644
index 00000000000..3d91af9ec51
--- /dev/null
+++ b/cache_service/migrations/sql/20260408110000_init_schema.sql
@@ -0,0 +1,22 @@
+-- Cache service initial schema
+
+CREATE TABLE IF NOT EXISTS cache_service_outputs (
+ key VARCHAR(512) PRIMARY KEY,
+ output_uri TEXT NOT NULL DEFAULT '',
+ metadata BYTEA,
+ last_updated TIMESTAMPTZ NOT NULL,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+CREATE INDEX IF NOT EXISTS idx_cache_service_outputs_last_updated ON cache_service_outputs (last_updated);
+
+CREATE TABLE IF NOT EXISTS cache_service_reservations (
+ key VARCHAR(512) PRIMARY KEY,
+ owner_id TEXT NOT NULL,
+ heartbeat_seconds INTEGER NOT NULL DEFAULT 0,
+ expires_at TIMESTAMPTZ NOT NULL,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+CREATE INDEX IF NOT EXISTS idx_cache_service_reservations_owner_id ON cache_service_reservations (owner_id);
+CREATE INDEX IF NOT EXISTS idx_cache_service_reservations_expires_at ON cache_service_reservations (expires_at);
diff --git a/cache_service/repository/errors/errors.go b/cache_service/repository/errors/errors.go
new file mode 100644
index 00000000000..388cd9f6161
--- /dev/null
+++ b/cache_service/repository/errors/errors.go
@@ -0,0 +1,23 @@
+package errors
+
+import (
+ "database/sql"
+ stderrors "errors"
+)
+
+var ErrReservationNotClaimable = stderrors.New("reservation not claimable")
+
+// ErrAlreadyExists is returned when an insert violates a unique constraint.
+var ErrAlreadyExists = stderrors.New("already exists")
+
+func IsNotFound(err error) bool {
+ return stderrors.Is(err, sql.ErrNoRows)
+}
+
+func IsAlreadyExists(err error) bool {
+ return stderrors.Is(err, ErrAlreadyExists)
+}
+
+func IsReservationNotClaimable(err error) bool {
+ return stderrors.Is(err, ErrReservationNotClaimable)
+}
diff --git a/cache_service/repository/impl/cached_output.go b/cache_service/repository/impl/cached_output.go
new file mode 100644
index 00000000000..a0011df3cb8
--- /dev/null
+++ b/cache_service/repository/impl/cached_output.go
@@ -0,0 +1,57 @@
+package impl
+
+import (
+ "context"
+ "database/sql"
+
+ "github.com/jmoiron/sqlx"
+
+ "github.com/flyteorg/flyte/v2/cache_service/repository/interfaces"
+ "github.com/flyteorg/flyte/v2/cache_service/repository/models"
+)
+
+var _ interfaces.CachedOutputRepo = (*CachedOutputRepo)(nil)
+
+type CachedOutputRepo struct {
+ db *sqlx.DB
+}
+
+func NewCachedOutputRepo(db *sqlx.DB) *CachedOutputRepo {
+ return &CachedOutputRepo{db: db}
+}
+
+func (r *CachedOutputRepo) Get(ctx context.Context, key string) (*models.CachedOutput, error) {
+ var output models.CachedOutput
+ err := sqlx.GetContext(ctx, r.db, &output,
+ "SELECT * FROM cache_service_outputs WHERE key = $1", key)
+ if err != nil {
+ return nil, err
+ }
+ return &output, nil
+}
+
+func (r *CachedOutputRepo) Put(ctx context.Context, output *models.CachedOutput) error {
+ _, err := r.db.ExecContext(ctx,
+ `INSERT INTO cache_service_outputs (key, output_uri, metadata, last_updated, created_at, updated_at)
+ VALUES ($1, $2, $3, $4, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
+ ON CONFLICT (key) DO UPDATE SET
+ output_uri = EXCLUDED.output_uri,
+ metadata = EXCLUDED.metadata,
+ last_updated = EXCLUDED.last_updated,
+ updated_at = CURRENT_TIMESTAMP`,
+ output.Key, output.OutputURI, output.Metadata, output.LastUpdated)
+ return err
+}
+
+func (r *CachedOutputRepo) Delete(ctx context.Context, key string) error {
+ result, err := r.db.ExecContext(ctx,
+ "DELETE FROM cache_service_outputs WHERE key = $1", key)
+ if err != nil {
+ return err
+ }
+ rows, _ := result.RowsAffected()
+ if rows == 0 {
+ return sql.ErrNoRows
+ }
+ return nil
+}
diff --git a/cache_service/repository/impl/reservation.go b/cache_service/repository/impl/reservation.go
new file mode 100644
index 00000000000..8becb21b5ec
--- /dev/null
+++ b/cache_service/repository/impl/reservation.go
@@ -0,0 +1,80 @@
+package impl
+
+import (
+ "context"
+ "database/sql"
+ "time"
+
+ "github.com/jmoiron/sqlx"
+
+ repositoryerrors "github.com/flyteorg/flyte/v2/cache_service/repository/errors"
+ "github.com/flyteorg/flyte/v2/cache_service/repository/interfaces"
+ "github.com/flyteorg/flyte/v2/cache_service/repository/models"
+)
+
+var _ interfaces.ReservationRepo = (*ReservationRepo)(nil)
+
+type ReservationRepo struct {
+ db *sqlx.DB
+}
+
+func NewReservationRepo(db *sqlx.DB) *ReservationRepo {
+ return &ReservationRepo{db: db}
+}
+
+func (r *ReservationRepo) Get(ctx context.Context, key string) (*models.Reservation, error) {
+ var reservation models.Reservation
+ err := sqlx.GetContext(ctx, r.db, &reservation,
+ "SELECT * FROM cache_service_reservations WHERE key = $1", key)
+ if err != nil {
+ return nil, err
+ }
+ return &reservation, nil
+}
+
+func (r *ReservationRepo) Create(ctx context.Context, reservation *models.Reservation) error {
+ result, err := r.db.ExecContext(ctx,
+ `INSERT INTO cache_service_reservations (key, owner_id, heartbeat_seconds, expires_at, created_at, updated_at)
+ VALUES ($1, $2, $3, $4, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
+ ON CONFLICT (key) DO NOTHING`,
+ reservation.Key, reservation.OwnerID, reservation.HeartbeatSeconds, reservation.ExpiresAt)
+ if err != nil {
+ return err
+ }
+ rows, _ := result.RowsAffected()
+ if rows == 0 {
+ return repositoryerrors.ErrAlreadyExists
+ }
+ return nil
+}
+
+func (r *ReservationRepo) UpdateIfExpiredOrOwned(ctx context.Context, reservation *models.Reservation, now time.Time) error {
+ result, err := r.db.ExecContext(ctx,
+ `UPDATE cache_service_reservations
+ SET owner_id = $1, heartbeat_seconds = $2, expires_at = $3, updated_at = $4
+ WHERE key = $5 AND (expires_at <= $6 OR owner_id = $7)`,
+ reservation.OwnerID, reservation.HeartbeatSeconds, reservation.ExpiresAt, now,
+ reservation.Key, now, reservation.OwnerID)
+ if err != nil {
+ return err
+ }
+ rows, _ := result.RowsAffected()
+ if rows == 0 {
+ return repositoryerrors.ErrReservationNotClaimable
+ }
+ return nil
+}
+
+func (r *ReservationRepo) DeleteByKeyAndOwner(ctx context.Context, key, ownerID string) error {
+ result, err := r.db.ExecContext(ctx,
+ "DELETE FROM cache_service_reservations WHERE key = $1 AND owner_id = $2",
+ key, ownerID)
+ if err != nil {
+ return err
+ }
+ rows, _ := result.RowsAffected()
+ if rows == 0 {
+ return sql.ErrNoRows
+ }
+ return nil
+}
diff --git a/cache_service/repository/interfaces/cached_output.go b/cache_service/repository/interfaces/cached_output.go
new file mode 100644
index 00000000000..f9f68cc80bd
--- /dev/null
+++ b/cache_service/repository/interfaces/cached_output.go
@@ -0,0 +1,13 @@
+package interfaces
+
+import (
+ "context"
+
+ "github.com/flyteorg/flyte/v2/cache_service/repository/models"
+)
+
+type CachedOutputRepo interface {
+ Get(ctx context.Context, key string) (*models.CachedOutput, error)
+ Put(ctx context.Context, output *models.CachedOutput) error
+ Delete(ctx context.Context, key string) error
+}
diff --git a/cache_service/repository/interfaces/repository.go b/cache_service/repository/interfaces/repository.go
new file mode 100644
index 00000000000..0ce3f41509c
--- /dev/null
+++ b/cache_service/repository/interfaces/repository.go
@@ -0,0 +1,6 @@
+package interfaces
+
+type Repository interface {
+ CachedOutputRepo() CachedOutputRepo
+ ReservationRepo() ReservationRepo
+}
diff --git a/cache_service/repository/interfaces/reservation.go b/cache_service/repository/interfaces/reservation.go
new file mode 100644
index 00000000000..e417f49f16d
--- /dev/null
+++ b/cache_service/repository/interfaces/reservation.go
@@ -0,0 +1,15 @@
+package interfaces
+
+import (
+ "context"
+ "time"
+
+ "github.com/flyteorg/flyte/v2/cache_service/repository/models"
+)
+
+type ReservationRepo interface {
+ Get(ctx context.Context, key string) (*models.Reservation, error)
+ Create(ctx context.Context, reservation *models.Reservation) error
+ UpdateIfExpiredOrOwned(ctx context.Context, reservation *models.Reservation, now time.Time) error
+ DeleteByKeyAndOwner(ctx context.Context, key, ownerID string) error
+}
diff --git a/cache_service/repository/models/cached_output.go b/cache_service/repository/models/cached_output.go
new file mode 100644
index 00000000000..801659801cc
--- /dev/null
+++ b/cache_service/repository/models/cached_output.go
@@ -0,0 +1,12 @@
+package models
+
+import "time"
+
+type CachedOutput struct {
+ Key string `db:"key"`
+ OutputURI string `db:"output_uri"`
+ Metadata []byte `db:"metadata"`
+ LastUpdated time.Time `db:"last_updated"`
+ CreatedAt time.Time `db:"created_at"`
+ UpdatedAt time.Time `db:"updated_at"`
+}
diff --git a/cache_service/repository/models/reservation.go b/cache_service/repository/models/reservation.go
new file mode 100644
index 00000000000..90eec89b011
--- /dev/null
+++ b/cache_service/repository/models/reservation.go
@@ -0,0 +1,12 @@
+package models
+
+import "time"
+
+type Reservation struct {
+ Key string `db:"key"`
+ OwnerID string `db:"owner_id"`
+ HeartbeatSeconds int64 `db:"heartbeat_seconds"`
+ ExpiresAt time.Time `db:"expires_at"`
+ CreatedAt time.Time `db:"created_at"`
+ UpdatedAt time.Time `db:"updated_at"`
+}
diff --git a/cache_service/repository/repository.go b/cache_service/repository/repository.go
new file mode 100644
index 00000000000..f87d39b3165
--- /dev/null
+++ b/cache_service/repository/repository.go
@@ -0,0 +1,28 @@
+package repository
+
+import (
+ "github.com/jmoiron/sqlx"
+
+ "github.com/flyteorg/flyte/v2/cache_service/repository/impl"
+ "github.com/flyteorg/flyte/v2/cache_service/repository/interfaces"
+)
+
+type repository struct {
+ cachedOutputRepo interfaces.CachedOutputRepo
+ reservationRepo interfaces.ReservationRepo
+}
+
+func NewRepository(db *sqlx.DB) interfaces.Repository {
+ return &repository{
+ cachedOutputRepo: impl.NewCachedOutputRepo(db),
+ reservationRepo: impl.NewReservationRepo(db),
+ }
+}
+
+func (r *repository) CachedOutputRepo() interfaces.CachedOutputRepo {
+ return r.cachedOutputRepo
+}
+
+func (r *repository) ReservationRepo() interfaces.ReservationRepo {
+ return r.reservationRepo
+}
diff --git a/cache_service/service/service.go b/cache_service/service/service.go
new file mode 100644
index 00000000000..fe7088a235f
--- /dev/null
+++ b/cache_service/service/service.go
@@ -0,0 +1,176 @@
+package service
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "connectrpc.com/connect"
+ cacheconfig "github.com/flyteorg/flyte/v2/cache_service/config"
+ "github.com/flyteorg/flyte/v2/cache_service/manager"
+ "github.com/flyteorg/flyte/v2/cache_service/repository"
+ cacheservicepb "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/cacheservice"
+ cacheservicev2 "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/cacheservice/v2"
+ "github.com/jmoiron/sqlx"
+)
+
+type CacheService struct {
+ manager *manager.Manager
+}
+
+func NewCacheService(cfg *cacheconfig.Config, db *sqlx.DB) *CacheService {
+ repos := repository.NewRepository(db)
+ return &CacheService{
+ manager: manager.New(
+ cfg,
+ repos.CachedOutputRepo(),
+ repos.ReservationRepo(),
+ ),
+ }
+}
+
+func (s *CacheService) Get(ctx context.Context, req *connect.Request[cacheservicev2.GetCacheRequest]) (*connect.Response[cacheservicepb.GetCacheResponse], error) {
+ if err := validateRequest(req.Msg); err != nil {
+ return nil, err
+ }
+ base := req.Msg.GetBaseRequest()
+ if base == nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("base_request is required"))
+ }
+ if err := validateBaseKey(base.GetKey()); err != nil {
+ return nil, err
+ }
+ entry, err := s.manager.Get(ctx, &cacheservicepb.GetCacheRequest{Key: scopedKey(base.GetKey(), req.Msg.GetIdentifier())})
+ if err != nil {
+ return nil, err
+ }
+ return connect.NewResponse(&cacheservicepb.GetCacheResponse{
+ Output: &cacheservicepb.CachedOutput{
+ Output: &cacheservicepb.CachedOutput_OutputUri{OutputUri: entry.OutputURI},
+ Metadata: entry.Metadata,
+ },
+ }), nil
+}
+
+func (s *CacheService) Put(ctx context.Context, req *connect.Request[cacheservicev2.PutCacheRequest]) (*connect.Response[cacheservicepb.PutCacheResponse], error) {
+ if err := validateRequest(req.Msg); err != nil {
+ return nil, err
+ }
+ base := req.Msg.GetBaseRequest()
+ if base == nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("base_request is required"))
+ }
+ if err := validateBaseKey(base.GetKey()); err != nil {
+ return nil, err
+ }
+ if base.GetOutput() == nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("base_request.output is required"))
+ }
+ managerReq := &cacheservicepb.PutCacheRequest{
+ Key: scopedKey(base.GetKey(), req.Msg.GetIdentifier()),
+ Output: base.GetOutput(),
+ Overwrite: base.GetOverwrite(),
+ }
+ if err := s.manager.Put(ctx, managerReq); err != nil {
+ return nil, err
+ }
+ return connect.NewResponse(&cacheservicepb.PutCacheResponse{}), nil
+}
+
+func (s *CacheService) Delete(ctx context.Context, req *connect.Request[cacheservicev2.DeleteCacheRequest]) (*connect.Response[cacheservicepb.DeleteCacheResponse], error) {
+ if err := validateRequest(req.Msg); err != nil {
+ return nil, err
+ }
+ base := req.Msg.GetBaseRequest()
+ if base == nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("base_request is required"))
+ }
+ if err := validateBaseKey(base.GetKey()); err != nil {
+ return nil, err
+ }
+ if err := s.manager.Delete(ctx, &cacheservicepb.DeleteCacheRequest{Key: scopedKey(base.GetKey(), req.Msg.GetIdentifier())}); err != nil {
+ return nil, err
+ }
+ return connect.NewResponse(&cacheservicepb.DeleteCacheResponse{}), nil
+}
+
+func (s *CacheService) GetOrExtendReservation(ctx context.Context, req *connect.Request[cacheservicev2.GetOrExtendReservationRequest]) (*connect.Response[cacheservicepb.GetOrExtendReservationResponse], error) {
+ if err := validateRequest(req.Msg); err != nil {
+ return nil, err
+ }
+ base := req.Msg.GetBaseRequest()
+ if base == nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("base_request is required"))
+ }
+ if err := validateBaseKey(base.GetKey()); err != nil {
+ return nil, err
+ }
+ if err := validateOwnerID(base.GetOwnerId()); err != nil {
+ return nil, err
+ }
+ managerReq := &cacheservicepb.GetOrExtendReservationRequest{
+ Key: scopedKey(base.GetKey(), req.Msg.GetIdentifier()),
+ OwnerId: base.GetOwnerId(),
+ HeartbeatInterval: base.GetHeartbeatInterval(),
+ }
+ reservation, err := s.manager.GetOrExtendReservation(ctx, managerReq, time.Now().UTC())
+ if err != nil {
+ return nil, err
+ }
+ return connect.NewResponse(&cacheservicepb.GetOrExtendReservationResponse{
+ Reservation: reservation,
+ }), nil
+}
+
+func (s *CacheService) ReleaseReservation(ctx context.Context, req *connect.Request[cacheservicev2.ReleaseReservationRequest]) (*connect.Response[cacheservicepb.ReleaseReservationResponse], error) {
+ if err := validateRequest(req.Msg); err != nil {
+ return nil, err
+ }
+ base := req.Msg.GetBaseRequest()
+ if base == nil {
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("base_request is required"))
+ }
+ if err := validateBaseKey(base.GetKey()); err != nil {
+ return nil, err
+ }
+ if err := validateOwnerID(base.GetOwnerId()); err != nil {
+ return nil, err
+ }
+ managerReq := &cacheservicepb.ReleaseReservationRequest{
+ Key: scopedKey(base.GetKey(), req.Msg.GetIdentifier()),
+ OwnerId: base.GetOwnerId(),
+ }
+ if err := s.manager.ReleaseReservation(ctx, managerReq); err != nil {
+ return nil, err
+ }
+ return connect.NewResponse(&cacheservicepb.ReleaseReservationResponse{}), nil
+}
+
+type validatableRequest interface {
+ Validate() error
+}
+
+func validateRequest(msg validatableRequest) error {
+ if err := msg.Validate(); err != nil {
+ return connect.NewError(connect.CodeInvalidArgument, err)
+ }
+ return nil
+}
+
+func validateBaseKey(key string) error {
+ if key == "" {
+ return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("base_request.key is required"))
+ }
+ return nil
+}
+
+func validateOwnerID(ownerID string) error {
+ if ownerID == "" {
+ return connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("base_request.owner_id is required"))
+ }
+ return nil
+}
+
+func scopedKey(key string, id *cacheservicev2.Identifier) string {
+ return fmt.Sprintf("%s-%s-%s", id.GetProject(), id.GetDomain(), key)
+}
diff --git a/cache_service/setup.go b/cache_service/setup.go
new file mode 100644
index 00000000000..9eb3225881c
--- /dev/null
+++ b/cache_service/setup.go
@@ -0,0 +1,39 @@
+package cache_service
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/app"
+ "github.com/flyteorg/flyte/v2/cache_service/config"
+ "github.com/flyteorg/flyte/v2/cache_service/migrations"
+ "github.com/flyteorg/flyte/v2/cache_service/service"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ v2connect "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/cacheservice/v2/v2connect"
+)
+
+// Setup registers the CacheService handler on the SetupContext mux.
+// Requires sc.DB and sc.DataStore to be set by the standalone binary.
+func Setup(ctx context.Context, sc *app.SetupContext) error {
+ cfg := config.GetConfig()
+ if err := migrations.RunMigrations(ctx, sc.DB); err != nil {
+ return fmt.Errorf("cache_service: failed to run migrations: %w", err)
+ }
+
+ path, handler := v2connect.NewCacheServiceHandler(service.NewCacheService(cfg, sc.DB))
+ sc.Mux.Handle(path, handler)
+ logger.Infof(ctx, "Mounted CacheService at %s", path)
+
+ sc.AddReadyCheck(func(r *http.Request) error {
+ if err := sc.DB.PingContext(r.Context()); err != nil {
+ return fmt.Errorf("database ping failed: %w", err)
+ }
+ if sc.DataStore.GetBaseContainerFQN(r.Context()) == "" {
+ return fmt.Errorf("storage connection error")
+ }
+ return nil
+ })
+
+ return nil
+}
diff --git a/charts/.gitignore b/charts/.gitignore
new file mode 100644
index 00000000000..ee3892e8794
--- /dev/null
+++ b/charts/.gitignore
@@ -0,0 +1 @@
+charts/
diff --git a/charts/flyte-binary/.gitignore b/charts/flyte-binary/.gitignore
new file mode 100644
index 00000000000..4f62b849d56
--- /dev/null
+++ b/charts/flyte-binary/.gitignore
@@ -0,0 +1 @@
+gen
diff --git a/charts/flyte-binary/.helmignore b/charts/flyte-binary/.helmignore
new file mode 100644
index 00000000000..0e8a0eb36f4
--- /dev/null
+++ b/charts/flyte-binary/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/charts/flyte-binary/Chart.yaml b/charts/flyte-binary/Chart.yaml
new file mode 100644
index 00000000000..11f4da37322
--- /dev/null
+++ b/charts/flyte-binary/Chart.yaml
@@ -0,0 +1,11 @@
+apiVersion: v2
+name: flyte-binary
+description: Chart for basic single Flyte executable deployment
+
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+
+version: v0.2.0 # VERSION
diff --git a/charts/flyte-binary/Makefile b/charts/flyte-binary/Makefile
new file mode 100644
index 00000000000..2f175d4c468
--- /dev/null
+++ b/charts/flyte-binary/Makefile
@@ -0,0 +1,17 @@
+
+GENERATED_FILE="gen/binary_manifest.yaml"
+DEVBOX_GENERATED_FILE="gen/devboxmanifest.yaml"
+
+cleanhelm:
+ @[ -f $(GENERATED_FILE) ] && rm $(GENERATED_FILE) || true
+
+cleandevbox:
+ @[ -f $(DEVBOX_GENERATED_FILE) ] && rm $(DEVBOX_GENERATED_FILE) || true
+
+.PHONY: helm
+helm: cleanhelm
+ helm template binary-tst-deploy ./ -n flyte --dependency-update --debug --create-namespace -a rbac.authorization.k8s.io/v1 -a networking.k8s.io/v1/Ingress -a apiextensions.k8s.io/v1/CustomResourceDefinition > $(GENERATED_FILE)
+
+.PHONY: devboxhelm
+devboxhelm: cleandevbox
+ helm template flytedevbox ./ -f flytectldemo.yaml -n flyte --dependency-update --debug --create-namespace -a rbac.authorization.k8s.io/v1 -a networking.k8s.io/v1/Ingress -a apiextensions.k8s.io/v1/CustomResourceDefinition > $(DEVBOX_GENERATED_FILE)
diff --git a/charts/flyte-binary/README.md b/charts/flyte-binary/README.md
new file mode 100644
index 00000000000..fb31f8b9cea
--- /dev/null
+++ b/charts/flyte-binary/README.md
@@ -0,0 +1,175 @@
+# flyte-binary
+
+  
+
+Chart for basic single Flyte executable deployment
+
+## Requirements
+
+| Repository | Name | Version |
+|------------|------|---------|
+| file://../flyteconnector | flyteconnector(flyteconnector) | v0.1.10 |
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| clusterResourceTemplates.annotations | object | `{}` | |
+| clusterResourceTemplates.externalConfigMap | string | `""` | |
+| clusterResourceTemplates.inline | object | `{}` | |
+| clusterResourceTemplates.inlineConfigMap | string | `""` | |
+| clusterResourceTemplates.labels | object | `{}` | |
+| commonAnnotations | object | `{}` | |
+| commonLabels | object | `{}` | |
+| configuration.annotations | object | `{}` | |
+| configuration.auth.authorizedUris | list | `[]` | |
+| configuration.auth.clientSecretsExternalSecretRef | string | `""` | |
+| configuration.auth.enableAuthServer | bool | `true` | |
+| configuration.auth.enabled | bool | `false` | |
+| configuration.auth.flyteClient.audience | string | `""` | |
+| configuration.auth.flyteClient.clientId | string | `"flytectl"` | |
+| configuration.auth.flyteClient.redirectUri | string | `"http://localhost:53593/callback"` | |
+| configuration.auth.flyteClient.scopes[0] | string | `"all"` | |
+| configuration.auth.internal.clientId | string | `"flytepropeller"` | |
+| configuration.auth.internal.clientSecret | string | `""` | |
+| configuration.auth.internal.clientSecretHash | string | `""` | |
+| configuration.auth.oidc.baseUrl | string | `""` | |
+| configuration.auth.oidc.clientId | string | `""` | |
+| configuration.auth.oidc.clientSecret | string | `""` | |
+| configuration.co-pilot.image.repository | string | `"cr.flyte.org/flyteorg/flytecopilot"` | |
+| configuration.co-pilot.image.tag | string | `"v1.16.4"` | |
+| configuration.connectorService.defaultConnector.defaultTimeout | string | `"10s"` | |
+| configuration.connectorService.defaultConnector.endpoint | string | `"k8s://flyteconnector.flyte:8000"` | |
+| configuration.connectorService.defaultConnector.insecure | bool | `true` | |
+| configuration.connectorService.defaultConnector.timeouts.GetTask | string | `"10s"` | |
+| configuration.connectorService.defaultConnector.timeouts.ListAgents | string | `"3s"` | |
+| configuration.database.dbname | string | `"flyte"` | |
+| configuration.database.host | string | `"127.0.0.1"` | |
+| configuration.database.options | string | `"sslmode=disable"` | |
+| configuration.database.password | string | `""` | |
+| configuration.database.passwordPath | string | `""` | |
+| configuration.database.port | int | `5432` | |
+| configuration.database.username | string | `"postgres"` | |
+| configuration.externalConfigMap | string | `""` | |
+| configuration.externalSecretRef | string | `""` | |
+| configuration.inline | object | `{}` | |
+| configuration.inlineConfigMap | string | `""` | |
+| configuration.inlineSecretRef | string | `""` | |
+| configuration.labels | object | `{}` | |
+| configuration.logging.level | int | `1` | |
+| configuration.logging.plugins.cloudwatch.enabled | bool | `false` | |
+| configuration.logging.plugins.cloudwatch.templateUri | string | `""` | |
+| configuration.logging.plugins.custom | list | `[]` | |
+| configuration.logging.plugins.kubernetes.enabled | bool | `false` | |
+| configuration.logging.plugins.kubernetes.templateUri | string | `""` | |
+| configuration.logging.plugins.stackdriver.enabled | bool | `false` | |
+| configuration.logging.plugins.stackdriver.templateUri | string | `""` | |
+| configuration.propeller.createCRDs | bool | `true` | |
+| configuration.propeller.literalOffloadingConfigEnabled | bool | `false` | |
+| configuration.storage.metadataContainer | string | `"my-organization-flyte-container"` | |
+| configuration.storage.provider | string | `"s3"` | |
+| configuration.storage.providerConfig.azure.account | string | `"storage-account-name"` | |
+| configuration.storage.providerConfig.azure.configDomainSuffix | string | `""` | |
+| configuration.storage.providerConfig.azure.configUploadConcurrency | int | `4` | |
+| configuration.storage.providerConfig.azure.key | string | `""` | |
+| configuration.storage.providerConfig.gcs.project | string | `"my-organization-gcp-project"` | |
+| configuration.storage.providerConfig.s3.accessKey | string | `""` | |
+| configuration.storage.providerConfig.s3.authType | string | `"iam"` | |
+| configuration.storage.providerConfig.s3.disableSSL | bool | `false` | |
+| configuration.storage.providerConfig.s3.endpoint | string | `""` | |
+| configuration.storage.providerConfig.s3.region | string | `"us-east-1"` | |
+| configuration.storage.providerConfig.s3.secretKey | string | `""` | |
+| configuration.storage.providerConfig.s3.v2Signing | bool | `false` | |
+| configuration.storage.userDataContainer | string | `"my-organization-flyte-container"` | |
+| deployment.annotations | object | `{}` | |
+| deployment.args | list | `[]` | |
+| deployment.command | list | `[]` | |
+| deployment.extraEnvVars | list | `[]` | |
+| deployment.extraEnvVarsConfigMap | string | `""` | |
+| deployment.extraEnvVarsSecret | string | `""` | |
+| deployment.extraPodSpec | object | `{}` | |
+| deployment.extraVolumeMounts | list | `[]` | |
+| deployment.extraVolumes | list | `[]` | |
+| deployment.genAdminAuthSecret.args | list | `[]` | |
+| deployment.genAdminAuthSecret.command | list | `[]` | |
+| deployment.genAdminAuthSecret.securityContext | object | `{}` | |
+| deployment.image.pullPolicy | string | `"IfNotPresent"` | |
+| deployment.image.repository | string | `"cr.flyte.org/flyteorg/flyte-binary"` | |
+| deployment.image.tag | string | `"latest"` | |
+| deployment.initContainers | list | `[]` | |
+| deployment.labels | object | `{}` | |
+| deployment.lifecycleHooks | object | `{}` | |
+| deployment.livenessProbe | object | `{}` | |
+| deployment.podAnnotations | object | `{}` | |
+| deployment.podLabels | object | `{}` | |
+| deployment.podSecurityContext.enabled | bool | `false` | |
+| deployment.podSecurityContext.fsGroup | int | `65534` | |
+| deployment.podSecurityContext.runAsGroup | int | `65534` | |
+| deployment.podSecurityContext.runAsUser | int | `65534` | |
+| deployment.preInitContainers | list | `[]` | |
+| deployment.readinessProbe | object | `{}` | |
+| deployment.securityContext | object | `{}` | |
+| deployment.sidecars | list | `[]` | |
+| deployment.startupProbe | object | `{}` | |
+| deployment.waitForDB.args | list | `[]` | |
+| deployment.waitForDB.command | list | `[]` | |
+| deployment.waitForDB.image.pullPolicy | string | `"IfNotPresent"` | |
+| deployment.waitForDB.image.repository | string | `"postgres"` | |
+| deployment.waitForDB.image.tag | string | `"15-alpine"` | |
+| deployment.waitForDB.securityContext | object | `{}` | |
+| enabled_plugins.tasks | object | `{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","connector-service","echo"]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) |
+| enabled_plugins.tasks.task-plugins | object | `{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","connector-service","echo"]}` | Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) |
+| enabled_plugins.tasks.task-plugins.enabled-plugins | list | `["container","sidecar","k8s-array","connector-service","echo"]` | [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend plugins |
+| flyte-core-components.admin.disableClusterResourceManager | bool | `false` | |
+| flyte-core-components.admin.disableScheduler | bool | `false` | |
+| flyte-core-components.admin.disabled | bool | `false` | |
+| flyte-core-components.admin.seedProjectsWithDetails[0].description | string | `"Default project setup."` | |
+| flyte-core-components.admin.seedProjectsWithDetails[0].name | string | `"flytesnacks"` | |
+| flyte-core-components.admin.seedProjects[0] | string | `"flytesnacks"` | |
+| flyte-core-components.dataCatalog.disabled | bool | `false` | |
+| flyte-core-components.propeller.disableWebhook | bool | `false` | |
+| flyte-core-components.propeller.disabled | bool | `false` | |
+| flyteconnector.enabled | bool | `false` | |
+| fullnameOverride | string | `""` | |
+| ingress.commonAnnotations | object | `{}` | |
+| ingress.create | bool | `false` | |
+| ingress.grpcAnnotations | object | `{}` | |
+| ingress.grpcExtraPaths.append | list | `[]` | |
+| ingress.grpcExtraPaths.prepend | list | `[]` | |
+| ingress.grpcIngressClassName | string | `""` | |
+| ingress.grpcTls | list | `[]` | |
+| ingress.host | string | `""` | |
+| ingress.httpAnnotations | object | `{}` | |
+| ingress.httpExtraPaths.append | list | `[]` | |
+| ingress.httpExtraPaths.prepend | list | `[]` | |
+| ingress.httpIngressClassName | string | `""` | |
+| ingress.httpTls | list | `[]` | |
+| ingress.ingressClassName | string | `""` | |
+| ingress.labels | object | `{}` | |
+| ingress.separateGrpcIngress | bool | `true` | |
+| ingress.tls | list | `[]` | |
+| nameOverride | string | `""` | |
+| rbac.annotations | object | `{}` | |
+| rbac.create | bool | `true` | |
+| rbac.extraRules | list | `[]` | |
+| rbac.labels | object | `{}` | |
+| service.clusterIP | string | `""` | |
+| service.commonAnnotations | object | `{}` | |
+| service.externalTrafficPolicy | string | `"Cluster"` | |
+| service.extraPorts | list | `[]` | |
+| service.grpcAnnotations | object | `{}` | |
+| service.httpAnnotations | object | `{}` | |
+| service.labels | object | `{}` | |
+| service.loadBalancerIP | string | `""` | |
+| service.loadBalancerSourceRanges | list | `[]` | |
+| service.nodePorts.grpc | string | `""` | |
+| service.nodePorts.http | string | `""` | |
+| service.ports.grpc | string | `""` | |
+| service.ports.http | string | `""` | |
+| service.type | string | `"ClusterIP"` | |
+| serviceAccount.annotations | object | `{}` | |
+| serviceAccount.create | bool | `true` | |
+| serviceAccount.imagePullSecrets | list | `[]` | |
+| serviceAccount.labels | object | `{}` | |
+| serviceAccount.name | string | `""` | |
+
diff --git a/charts/flyte-binary/defaults/cluster-resource-templates/namespace.yaml b/charts/flyte-binary/defaults/cluster-resource-templates/namespace.yaml
new file mode 100644
index 00000000000..301cb82f42f
--- /dev/null
+++ b/charts/flyte-binary/defaults/cluster-resource-templates/namespace.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: '{{ namespace }}'
diff --git a/charts/flyte-binary/templates/NOTES.txt b/charts/flyte-binary/templates/NOTES.txt
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/charts/flyte-binary/templates/_helpers.tpl b/charts/flyte-binary/templates/_helpers.tpl
new file mode 100644
index 00000000000..6349140b802
--- /dev/null
+++ b/charts/flyte-binary/templates/_helpers.tpl
@@ -0,0 +1,188 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "flyte-binary.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "flyte-binary.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "flyte-binary.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Base labels
+*/}}
+{{- define "flyte-binary.baseLabels" -}}
+app.kubernetes.io/name: {{ include "flyte-binary.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "flyte-binary.labels" -}}
+helm.sh/chart: {{ include "flyte-binary.chart" . }}
+{{ include "flyte-binary.baseLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "flyte-binary.selectorLabels" -}}
+{{ include "flyte-binary.baseLabels" . }}
+app.kubernetes.io/component: flyte-binary
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "flyte-binary.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "flyte-binary.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Flag to use external configuration.
+*/}}
+{{- define "flyte-binary.configuration.externalConfiguration" -}}
+{{- or .Values.configuration.externalConfigMap .Values.configuration.externalSecretRef -}}
+{{- end -}}
+
+{{/*
+Get the Flyte configuration ConfigMap name.
+*/}}
+{{- define "flyte-binary.configuration.configMapName" -}}
+{{- printf "%s-config" (include "flyte-binary.fullname" .) -}}
+{{- end -}}
+
+{{/*
+Get the Flyte configuration Secret name.
+*/}}
+{{- define "flyte-binary.configuration.configSecretName" -}}
+{{- printf "%s-config-secret" (include "flyte-binary.fullname" .) -}}
+{{- end -}}
+
+{{/*
+Get the Flyte logging configuration.
+*/}}
+{{- define "flyte-binary.configuration.logging.plugins" -}}
+{{- with .Values.configuration.logging.plugins -}}
+kubernetes-enabled: {{ .kubernetes.enabled }}
+{{- if .kubernetes.enabled }}
+kubernetes-template-uri: {{ required "Template URI required for Kubernetes logging plugin" .kubernetes.templateUri }}
+{{- end }}
+cloudwatch-enabled: {{ .cloudwatch.enabled }}
+{{- if .cloudwatch.enabled }}
+cloudwatch-template-uri: {{ required "Template URI required for CloudWatch logging plugin" .cloudwatch.templateUri }}
+{{- end }}
+stackdriver-enabled: {{ .stackdriver.enabled }}
+{{- if .stackdriver.enabled }}
+stackdriver-template-uri: {{ required "Template URI required for stackdriver logging plugin" .stackdriver.templateUri }}
+{{- end }}
+{{- if .custom }}
+templates: {{- toYaml .custom | nindent 2 -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Get the Flyte cluster resource templates ConfigMap name.
+*/}}
+{{- define "flyte-binary.clusterResourceTemplates.configMapName" -}}
+{{- printf "%s-cluster-resource-templates" (include "flyte-binary.fullname" .) -}}
+{{- end -}}
+
+{{/*
+Get the Flyte HTTP service name
+*/}}
+{{- define "flyte-binary.service.http.name" -}}
+{{- printf "%s-http" (include "flyte-binary.fullname" .) -}}
+{{- end -}}
+
+{{/*
+Get the Flyte service HTTP port.
+*/}}
+{{- define "flyte-binary.service.http.port" -}}
+{{- default 8090 .Values.service.ports.http -}}
+{{- end -}}
+
+{{/*
+Get the Flyte gRPC service name
+*/}}
+{{- define "flyte-binary.service.grpc.name" -}}
+{{- printf "%s-http" (include "flyte-binary.fullname" .) -}}
+{{- end -}}
+
+{{/*
+Get the Flyte service gRPC port.
+*/}}
+{{- define "flyte-binary.service.grpc.port" -}}
+{{- default 8090 .Values.service.ports.grpc -}}
+{{- end -}}
+
+{{/*
+Get the Flyte API paths for ingress.
+*/}}
+{{- define "flyte-binary.ingress.grpcPaths" -}}
+- /flyteidl2.workflow.RunService
+- /flyteidl2.workflow.RunService/*
+- /flyteidl2.task.TaskService
+- /flyteidl2.task.TaskService/*
+- /flyteidl2.workflow.TranslatorService
+- /flyteidl2.workflow.TranslatorService/*
+- /flyteidl2.actions.ActionsService
+- /flyteidl2.actions.ActionsService/*
+- /flyteidl2.dataproxy.DataProxyService
+- /flyteidl2.dataproxy.DataProxyService/*
+- /flyteidl2.secret.SecretService
+- /flyteidl2.secret.SecretService/*
+{{- end -}}
+
+{{/*
+Get the Flyte webhook service name.
+*/}}
+{{- define "flyte-binary.webhook.serviceName" -}}
+{{- printf "%s-webhook" (include "flyte-binary.fullname" .) -}}
+{{- end -}}
+
+{{/*
+Get the Flyte webhook secret name.
+*/}}
+{{- define "flyte-binary.webhook.secretName" -}}
+{{- printf "%s-webhook-secret" (include "flyte-binary.fullname" .) -}}
+{{- end -}}
+
+{{/*
+Get the Flyte ClusterRole name.
+*/}}
+{{- define "flyte-binary.rbac.clusterRoleName" -}}
+{{- printf "%s-cluster-role" (include "flyte-binary.fullname" .) -}}
+{{- end -}}
diff --git a/charts/flyte-binary/templates/admin-auth-secret.yaml b/charts/flyte-binary/templates/admin-auth-secret.yaml
new file mode 100644
index 00000000000..3e164c1f109
--- /dev/null
+++ b/charts/flyte-binary/templates/admin-auth-secret.yaml
@@ -0,0 +1,16 @@
+{{- if .Values.configuration.auth.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: {{ include "flyte-binary.configuration.auth.adminAuthSecretName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-binary.labels" . | nindent 4 }}
+ {{- if .Values.commonLabels }}
+ {{- tpl ( .Values.commonLabels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- if .Values.commonAnnotations }}
+ {{- tpl ( .Values.commonAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+type: Opaque
+{{- end }}
diff --git a/charts/flyte-binary/templates/auth-client-secret.yaml b/charts/flyte-binary/templates/auth-client-secret.yaml
new file mode 100644
index 00000000000..05612d72a53
--- /dev/null
+++ b/charts/flyte-binary/templates/auth-client-secret.yaml
@@ -0,0 +1,19 @@
+{{- if and .Values.configuration.auth.enabled (not .Values.configuration.auth.clientSecretsExternalSecretRef) }}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: {{ include "flyte-binary.configuration.auth.clientSecretName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-binary.labels" . | nindent 4 }}
+ {{- if .Values.commonLabels }}
+ {{- tpl ( .Values.commonLabels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- if .Values.commonAnnotations }}
+ {{- tpl ( .Values.commonAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+type: Opaque
+stringData:
+ client_secret: {{ required "Internal client secret required when authentication is enabled" .Values.configuration.auth.internal.clientSecret | quote }}
+ oidc_client_secret: {{ required "OIDC client secret required when authentication is enabled" .Values.configuration.auth.oidc.clientSecret | quote }}
+{{- end }}
diff --git a/charts/flyte-binary/templates/clusterrole.yaml b/charts/flyte-binary/templates/clusterrole.yaml
new file mode 100644
index 00000000000..9999a292d63
--- /dev/null
+++ b/charts/flyte-binary/templates/clusterrole.yaml
@@ -0,0 +1,77 @@
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ include "flyte-binary.rbac.clusterRoleName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-binary.labels" . | nindent 4 }}
+ {{- if .Values.commonLabels }}
+ {{- tpl ( .Values.commonLabels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.rbac.labels }}
+ {{- tpl ( .Values.rbac.labels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- if .Values.commonAnnotations }}
+ {{- tpl ( .Values.commonAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.rbac.annotations }}
+ {{- tpl ( .Values.rbac.annotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+rules:
+ - apiGroups:
+ - ""
+ resources:
+ - pods
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+ - apiGroups:
+ - ""
+ resources:
+ - events
+ verbs:
+ - create
+ - delete
+ - patch
+ - update
+ - apiGroups:
+ - flyte.org
+ resources:
+ - taskactions
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+ - apiGroups:
+ - admissionregistration.k8s.io
+ resources:
+ - mutatingwebhookconfigurations
+ verbs:
+ - create
+ - get
+ - list
+ - patch
+ - update
+ - watch
+ - apiGroups:
+ - ""
+ resources:
+ - secrets
+ verbs:
+ - create
+ - get
+ - update
+ {{- if .Values.rbac.extraRules }}
+ {{- toYaml .Values.rbac.extraRules | nindent 2 }}
+ {{- end }}
+{{- end }}
diff --git a/charts/flyte-binary/templates/clusterrolebinding.yaml b/charts/flyte-binary/templates/clusterrolebinding.yaml
new file mode 100644
index 00000000000..3c13b302ad0
--- /dev/null
+++ b/charts/flyte-binary/templates/clusterrolebinding.yaml
@@ -0,0 +1,29 @@
+{{- if and .Values.rbac.create .Values.serviceAccount.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ include "flyte-binary.fullname" . }}-cluster-role-binding
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-binary.labels" . | nindent 4 }}
+ {{- if .Values.commonLabels }}
+ {{- tpl ( .Values.commonLabels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.rbac.labels }}
+ {{- tpl ( .Values.rbac.labels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- if .Values.commonAnnotations }}
+ {{- tpl ( .Values.commonAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.rbac.annotations }}
+ {{- tpl ( .Values.rbac.annotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ include "flyte-binary.rbac.clusterRoleName" . }}
+subjects:
+ - kind: ServiceAccount
+ name: {{ include "flyte-binary.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+{{- end }}
diff --git a/charts/flyte-binary/templates/config-secret.yaml b/charts/flyte-binary/templates/config-secret.yaml
new file mode 100644
index 00000000000..5992755b1fb
--- /dev/null
+++ b/charts/flyte-binary/templates/config-secret.yaml
@@ -0,0 +1,52 @@
+{{- if not (include "flyte-binary.configuration.externalConfiguration" .) }}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: {{ include "flyte-binary.configuration.configSecretName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-binary.labels" . | nindent 4 }}
+ {{- if .Values.commonLabels }}
+ {{- tpl ( .Values.commonLabels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.configuration.labels }}
+ {{- tpl ( .Values.configuration.labels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- if .Values.commonAnnotations }}
+ {{- tpl ( .Values.commonAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.configuration.annotations }}
+ {{- tpl ( .Values.configuration.annotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+type: Opaque
+stringData:
+ {{- if .Values.configuration.database.password }}
+ 012-database-secrets.yaml: |
+ database:
+ postgres:
+ password: {{ .Values.configuration.database.password | quote }}
+ {{- end }}
+ {{- if eq "s3" .Values.configuration.storage.provider }}
+ {{- if eq "accesskey" .Values.configuration.storage.providerConfig.s3.authType }}
+ 013-storage-secrets.yaml: |
+ storage:
+ stow:
+ config:
+ access_key_id: {{ required "Access key required for S3 storage provider" .Values.configuration.storage.providerConfig.s3.accessKey | quote }}
+ secret_key: {{ required "Secret key required for S3 storage provider" .Values.configuration.storage.providerConfig.s3.secretKey | quote }}
+ {{- end }}
+ {{- end }}
+ {{- if .Values.configuration.auth.enabled }}
+ {{- if .Values.configuration.auth.enableAuthServer }}
+ {{- if .Values.configuration.auth.internal.clientSecretHash }}
+ 014-auth-secrets.yaml: |
+ auth:
+ appAuth:
+ selfAuthServer:
+ staticClients:
+ flytepropeller:
+ client_secret: {{ .Values.configuration.auth.internal.clientSecretHash | quote }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+{{- end }}
diff --git a/charts/flyte-binary/templates/configmap.yaml b/charts/flyte-binary/templates/configmap.yaml
new file mode 100644
index 00000000000..e0d4f20a9b5
--- /dev/null
+++ b/charts/flyte-binary/templates/configmap.yaml
@@ -0,0 +1,112 @@
+{{- if not (include "flyte-binary.configuration.externalConfiguration" .) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "flyte-binary.configuration.configMapName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-binary.labels" . | nindent 4 }}
+ {{- if .Values.commonLabels }}
+ {{- tpl ( .Values.commonLabels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.configuration.labels }}
+ {{- tpl ( .Values.configuration.labels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- if .Values.commonAnnotations }}
+ {{- tpl ( .Values.commonAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.configuration.annotations }}
+ {{- tpl ( .Values.configuration.annotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+data:
+ 000-core.yaml: |
+ logger:
+ show-source: true
+ level: {{ default 1 .Values.configuration.logging.level }}
+ webhook:
+ certDir: /var/run/flyte/certs
+ localCert: true
+ secretName: {{ include "flyte-binary.webhook.secretName" . }}
+ serviceName: {{ include "flyte-binary.webhook.serviceName" . }}
+ servicePort: 443
+{{ tpl ( index .Values "flyte-core-components" | toYaml ) . | nindent 4 }}
+ 001-plugins.yaml: |
+ tasks:
+ {{- .Values.enabled_plugins.tasks | toYaml | nindent 6 }}
+ plugins:
+ logs: {{- include "flyte-binary.configuration.logging.plugins" . | nindent 8 }}
+ k8s:
+ co-pilot:
+ {{- with ( index .Values.configuration "co-pilot" ).image }}
+ image: {{ printf "%s:%s" .repository .tag | quote }}
+ {{- end }}
+ k8s-array:
+ logs:
+ config: {{- include "flyte-binary.configuration.logging.plugins" . | nindent 12 }}
+ 002-database.yaml: |
+ {{- with .Values.configuration.database }}
+ database:
+ postgres:
+ username: {{ .username }}
+ host: {{ tpl .host $ }}
+ port: {{ .port }}
+ dbname: {{ .dbname }}
+ options: {{ .options | quote }}
+ {{- if .passwordPath }}
+ passwordPath: {{ .passwordPath }}
+ {{- end }}
+ {{- end }}
+ 003-storage.yaml: |
+ {{- with .Values.configuration.storage }}
+ storage:
+ type: stow
+ stow:
+ {{- if eq "s3" .provider }}
+ {{- with .providerConfig.s3 }}
+ kind: s3
+ config:
+ region: {{ required "Region required for S3 storage provider" .region }}
+ disable_ssl: {{ .disableSSL }}
+ v2_signing: {{ .v2Signing }}
+ {{- if .endpoint }}
+ endpoint: {{ tpl .endpoint $ }}
+ {{- end }}
+ {{- if eq "iam" .authType }}
+ auth_type: iam
+ {{- else if eq "accesskey" .authType }}
+ auth_type: accesskey
+ {{- else }}
+ {{- printf "Invalid value for S3 storage provider authentication type. Expected one of (iam, accesskey), but got: %s" .authType | fail }}
+ {{- end }}
+ {{- end }}
+ {{- else if eq "azure" .provider }}
+ {{- with .providerConfig.azure }}
+ kind: azure
+ config:
+ account: {{ .account }}
+ {{- if .key }}
+ key: {{ .key }}
+ {{- end }}
+ {{- if .configDomainSuffix }}
+ configDomainSuffix: {{ .configDomainSuffix }}
+ {{- end }}
+ {{- if .configUploadConcurrency }}
+ configUploadConcurrency: {{ .configUploadConcurrency }}
+ {{- end }}
+ {{- end }}
+ {{- else if eq "gcs" .provider }}
+ kind: google
+ config:
+ json: ""
+ project_id: {{ required "GCP project required for GCS storage provider" .providerConfig.gcs.project }}
+ scopes: https://www.googleapis.com/auth/cloud-platform
+ {{- else }}
+ {{- printf "Invalid value for storage provider. Expected one of (s3, gcs), but got: %s" .provider | fail }}
+ {{- end }}
+ container: {{ required "Metadata container required" .metadataContainer }}
+ {{- end }}
+ {{- if .Values.configuration.inline }}
+ 100-inline-config.yaml: |
+ {{- tpl ( .Values.configuration.inline | toYaml ) . | nindent 4 }}
+ {{- end }}
+{{- end }}
diff --git a/charts/flyte-binary/templates/crds/flyte.org_taskactions.yaml b/charts/flyte-binary/templates/crds/flyte.org_taskactions.yaml
new file mode 100644
index 00000000000..969d28e00a8
--- /dev/null
+++ b/charts/flyte-binary/templates/crds/flyte.org_taskactions.yaml
@@ -0,0 +1,287 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+ annotations:
+ controller-gen.kubebuilder.io/version: v0.19.0
+ name: taskactions.flyte.org
+spec:
+ group: flyte.org
+ names:
+ kind: TaskAction
+ listKind: TaskActionList
+ plural: taskactions
+ singular: taskaction
+ scope: Namespaced
+ versions:
+ - additionalPrinterColumns:
+ - jsonPath: .spec.runName
+ name: Run
+ type: string
+ - jsonPath: .spec.actionName
+ name: Action
+ type: string
+ - jsonPath: .spec.taskType
+ name: TaskType
+ type: string
+ - jsonPath: .status.conditions[?(@.type=='Progressing')].reason
+ name: Status
+ type: string
+ - jsonPath: .metadata.creationTimestamp
+ name: Age
+ type: date
+ - jsonPath: .status.conditions[?(@.type=='Progressing')].status
+ name: Progressing
+ priority: 1
+ type: string
+ - jsonPath: .status.conditions[?(@.type=='Succeeded')].status
+ name: Succeeded
+ priority: 1
+ type: string
+ - jsonPath: .status.conditions[?(@.type=='Failed')].status
+ name: Failed
+ priority: 1
+ type: string
+ name: v1
+ schema:
+ openAPIV3Schema:
+ description: TaskAction is the Schema for the taskactions API
+ properties:
+ apiVersion:
+ description: |-
+ APIVersion defines the versioned schema of this representation of an object.
+ Servers should convert recognized schemas to the latest internal value, and
+ may reject unrecognized values.
+ More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+ type: string
+ kind:
+ description: |-
+ Kind is a string value representing the REST resource this object represents.
+ Servers may infer this from the endpoint the client submits requests to.
+ Cannot be updated.
+ In CamelCase.
+ More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+ type: string
+ metadata:
+ type: object
+ spec:
+ description: spec defines the desired state of TaskAction
+ properties:
+ actionName:
+ description: ActionName is the unique name of this action within the
+ run
+ maxLength: 30
+ minLength: 1
+ type: string
+ cacheKey:
+ description: |-
+ CacheKey enables cache lookup/writeback for this task action when set.
+ This is propagated from workflow.TaskAction.cache_key.
+ maxLength: 256
+ type: string
+ domain:
+ description: Domain this action belongs to
+ maxLength: 63
+ minLength: 1
+ type: string
+ envVars:
+ additionalProperties:
+ type: string
+ description: EnvVars are run-scoped environment variables projected
+ from RunSpec for executor runtime use.
+ type: object
+ group:
+ description: Group is the group this action belongs to, if applicable.
+ maxLength: 256
+ type: string
+ inputUri:
+ description: InputURI is the path to the input data for this action
+ minLength: 1
+ type: string
+ interruptible:
+ description: Interruptible is the run-scoped interruptibility override
+ projected from RunSpec.
+ type: boolean
+ parentActionName:
+ description: ParentActionName is the optional name of the parent action
+ maxLength: 30
+ minLength: 1
+ type: string
+ project:
+ description: Project this action belongs to
+ maxLength: 63
+ minLength: 1
+ type: string
+ runName:
+ description: RunName is the name of the run this action belongs to
+ maxLength: 30
+ minLength: 1
+ type: string
+ runOutputBase:
+ description: RunOutputBase is the base path where this action should
+ write its output
+ minLength: 1
+ type: string
+ shortName:
+ description: ShortName is the human-readable display name for this
+ task
+ maxLength: 63
+ type: string
+ taskTemplate:
+ description: TaskTemplate is the proto-serialized core.TaskTemplate
+ stored inline in etcd
+ format: byte
+ type: string
+ taskType:
+ description: TaskType identifies which plugin handles this task (e.g.
+ "container", "spark", "ray")
+ maxLength: 63
+ minLength: 1
+ type: string
+ required:
+ - actionName
+ - domain
+ - inputUri
+ - project
+ - runName
+ - runOutputBase
+ - taskTemplate
+ - taskType
+ type: object
+ status:
+ description: status defines the observed state of TaskAction
+ properties:
+ attempts:
+ description: Attempts is the latest observed action attempt number,
+ starting from 1.
+ format: int32
+ type: integer
+ cacheStatus:
+ description: CacheStatus is the latest observed cache lookup result
+ for this action.
+ format: int32
+ type: integer
+ conditions:
+ description: |-
+ conditions represent the current state of the TaskAction resource.
+ Each condition has a unique type and reflects the status of a specific aspect of the resource.
+
+ Standard condition types include:
+ - "Available": the resource is fully functional
+ - "Progressing": the resource is being created or updated
+ - "Degraded": the resource failed to reach or maintain its desired state
+
+ The status of each condition is one of True, False, or Unknown.
+ items:
+ description: Condition contains details for one aspect of the current
+ state of this API Resource.
+ properties:
+ lastTransitionTime:
+ description: |-
+ lastTransitionTime is the last time the condition transitioned from one status to another.
+ This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
+ format: date-time
+ type: string
+ message:
+ description: |-
+ message is a human readable message indicating details about the transition.
+ This may be an empty string.
+ maxLength: 32768
+ type: string
+ observedGeneration:
+ description: |-
+ observedGeneration represents the .metadata.generation that the condition was set based upon.
+ For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+ with respect to the current state of the instance.
+ format: int64
+ minimum: 0
+ type: integer
+ reason:
+ description: |-
+ reason contains a programmatic identifier indicating the reason for the condition's last transition.
+ Producers of specific condition types may define expected values and meanings for this field,
+ and whether the values are considered a guaranteed API.
+ The value should be a CamelCase string.
+ This field may not be empty.
+ maxLength: 1024
+ minLength: 1
+ pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+ type: string
+ status:
+ description: status of the condition, one of True, False, Unknown.
+ enum:
+ - "True"
+ - "False"
+ - Unknown
+ type: string
+ type:
+ description: type of condition in CamelCase or in foo.example.com/CamelCase.
+ maxLength: 316
+ pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+ type: string
+ required:
+ - lastTransitionTime
+ - message
+ - reason
+ - status
+ - type
+ type: object
+ type: array
+ x-kubernetes-list-map-keys:
+ - type
+ x-kubernetes-list-type: map
+ phaseHistory:
+ description: |-
+ PhaseHistory is an append-only log of phase transitions. Unlike conditions
+ (which are updated in-place by type), this preserves the full timeline:
+ Queued → Initializing → Executing → Succeeded/Failed, each with a timestamp.
+ items:
+ description: PhaseTransition records a phase change with its timestamp.
+ properties:
+ message:
+ description: Message is an optional human-readable message about
+ the transition.
+ type: string
+ occurredAt:
+ description: OccurredAt is when this phase transition happened.
+ format: date-time
+ type: string
+ phase:
+ description: Phase is the phase that was entered (e.g. "Queued",
+ "Initializing", "Executing", "Succeeded", "Failed").
+ type: string
+ required:
+ - occurredAt
+ - phase
+ type: object
+ type: array
+ pluginPhase:
+ description: PluginPhase is a human-readable representation of the
+ plugin's current phase.
+ type: string
+ pluginPhaseVersion:
+ description: PluginPhaseVersion is the version of the current plugin
+ phase.
+ format: int32
+ type: integer
+ pluginState:
+ description: PluginState is the Gob-encoded plugin state from the
+ last reconciliation round.
+ format: byte
+ type: string
+ pluginStateVersion:
+ description: PluginStateVersion tracks the version of the plugin state
+ schema for compatibility.
+ type: integer
+ stateJson:
+ description: StateJSON is the JSON serialized NodeStatus that was
+ last sent to the State Service
+ type: string
+ type: object
+ required:
+ - spec
+ type: object
+ served: true
+ storage: true
+ subresources:
+ status: {}
diff --git a/charts/flyte-binary/templates/deployment.yaml b/charts/flyte-binary/templates/deployment.yaml
new file mode 100644
index 00000000000..144c7c359d2
--- /dev/null
+++ b/charts/flyte-binary/templates/deployment.yaml
@@ -0,0 +1,222 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "flyte-binary.fullname" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-binary.labels" . | nindent 4 }}
+ {{- if .Values.commonLabels }}
+ {{- tpl ( .Values.commonLabels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.deployment.labels }}
+ {{- tpl ( .Values.deployment.labels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- if .Values.commonAnnotations }}
+ {{- tpl ( .Values.commonAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.deployment.annotations }}
+ {{- tpl ( .Values.deployment.annotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+spec:
+ replicas: 1
+ strategy:
+ type: Recreate
+ selector:
+ matchLabels: {{- include "flyte-binary.selectorLabels" . | nindent 6 }}
+ template:
+ metadata:
+ labels: {{- include "flyte-binary.selectorLabels" . | nindent 8 }}
+ {{- if .Values.commonLabels }}
+ {{- tpl ( .Values.commonLabels | toYaml ) . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.deployment.podLabels }}
+ {{- tpl ( .Values.deployment.podLabels | toYaml ) . | nindent 8 }}
+ {{- end }}
+ annotations:
+ {{- if not (include "flyte-binary.configuration.externalConfiguration" .) }}
+ checksum/configuration: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+ checksum/configuration-secret: {{ include (print $.Template.BasePath "/config-secret.yaml") . | sha256sum }}
+ {{- end }}
+ {{- if .Values.commonAnnotations }}
+ {{- tpl ( .Values.commonAnnotations | toYaml ) . | nindent 8 }}
+ {{- end }}
+ {{- if .Values.deployment.podAnnotations }}
+ {{- tpl ( .Values.deployment.podAnnotations | toYaml ) . | nindent 8 }}
+ {{- end }}
+ spec:
+ {{- if .Values.deployment.extraPodSpec }}
+ {{- tpl ( .Values.deployment.extraPodSpec | toYaml ) . | nindent 6 }}
+ {{- end }}
+ {{- if .Values.deployment.podSecurityContext.enabled }}
+ securityContext: {{- omit .Values.deployment.podSecurityContext "enabled" | toYaml | nindent 12 }}
+ {{- end }}
+ serviceAccountName: {{ include "flyte-binary.serviceAccountName" . }}
+ {{- if or .Values.deployment.initContainers (not (include "flyte-binary.configuration.externalConfiguration" .)) }}
+ initContainers:
+ {{- if not (include "flyte-binary.configuration.externalConfiguration" .) }}
+ {{- if .Values.deployment.preInitContainers }}
+ {{- tpl ( .Values.deployment.preInitContainers | toYaml ) . | nindent 8 }}
+ {{- end }}
+ - name: wait-for-db
+ {{- with .Values.deployment.waitForDB.image }}
+ image: {{ printf "%s:%s" .repository .tag | quote }}
+ imagePullPolicy: {{ .pullPolicy | quote }}
+ {{- end }}
+ command:
+ {{- if .Values.deployment.waitForDB.command }}
+ {{- tpl ( .Values.deployment.waitForDB.command | toYaml ) . | nindent 12 }}
+ {{- else }}
+ - sh
+ - -ec
+ {{- end }}
+ args:
+ {{- if .Values.deployment.waitForDB.args }}
+ {{- tpl ( .Values.deployment.waitForDB.args | toYaml ) . | nindent 12 }}
+ {{- else }}
+ {{- with .Values.configuration.database }}
+ - |
+ until pg_isready \
+ -h {{ tpl .host $ }} \
+ -p {{ .port }} \
+ -U {{ .username }}
+ do
+ echo waiting for database
+ sleep 0.1
+ done
+ {{- end }}
+ {{- end }}
+ {{- if .Values.deployment.resources }}
+ resources: {{- toYaml .Values.deployment.resources | nindent 12 }}
+ {{- end }}
+ {{- if .Values.deployment.waitForDB.securityContext }}
+ securityContext: {{- toYaml .Values.deployment.waitForDB.securityContext | nindent 12 }}
+ {{- end }}
+ {{- end }}
+ {{- if .Values.deployment.initContainers }}
+ {{- tpl ( .Values.deployment.initContainers | toYaml ) . | nindent 8 }}
+ {{- end }}
+ {{- end }}
+ containers:
+ - name: flyte
+ {{- with .Values.deployment.image }}
+ image: {{ printf "%s:%s" .repository .tag | quote }}
+ imagePullPolicy: {{ .pullPolicy | quote }}
+ {{- end }}
+ {{- if .Values.deployment.command }}
+ command: {{- tpl ( .Values.deployment.command | toYaml ) . | nindent 12 }}
+ {{- end }}
+ args:
+ {{- if .Values.deployment.args }}
+ {{- tpl ( .Values.deployment.args | toYaml ) . | nindent 12 }}
+ {{- else }}
+ - --config
+ - /etc/flyte/config.d/*.yaml
+ {{- end }}
+ env:
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ {{- if .Values.deployment.extraEnvVars }}
+ {{- tpl ( .Values.deployment.extraEnvVars | toYaml ) . | nindent 12 }}
+ {{- end }}
+ {{- if or .Values.deployment.extraEnvVarsConfigMap .Values.deployment.extraEnvVarsSecret }}
+ envFrom:
+ {{- if .Values.deployment.extraEnvVarsConfigMap }}
+ - configMapRef:
+ name: {{ .Values.deployment.extraEnvVarsConfigMap }}
+ {{- end }}
+ {{- if .Values.deployment.extraEnvVarsSecret }}
+ - secretRef:
+ name: {{ .Values.deployment.extraEnvVarsSecret }}
+ {{- end }}
+ {{- end }}
+ ports:
+ - name: http
+ containerPort: 8090
+ - name: webhook
+ containerPort: 9443
+ {{- if .Values.deployment.startupProbe }}
+ startupProbe: {{- tpl ( .Values.deployment.startupProbe | toYaml ) . | nindent 12 }}
+ {{- end }}
+ livenessProbe:
+ {{- if .Values.deployment.livenessProbe }}
+ {{- tpl ( .Values.deployment.livenessProbe | toYaml ) . | nindent 12 }}
+ {{- else }}
+ httpGet:
+ path: /healthz
+ port: http
+ initialDelaySeconds: 30
+ {{- end }}
+ readinessProbe:
+ {{- if .Values.deployment.readinessProbe }}
+ {{- tpl ( .Values.deployment.readinessProbe | toYaml ) . | nindent 12 }}
+ {{- else }}
+ httpGet:
+ path: /healthz
+ port: http
+ initialDelaySeconds: 30
+ {{- end }}
+ {{- if .Values.deployment.resources }}
+ resources: {{- toYaml .Values.deployment.resources | nindent 12 }}
+ {{- end }}
+ {{- if .Values.deployment.lifecycleHooks }}
+ lifecycle: {{- tpl ( .Values.deployment.lifecycleHooks | toYaml ) . | nindent 12 }}
+ {{- end }}
+ volumeMounts:
+ {{- if .Values.configuration.auth.enabled }}
+ - name: auth
+ mountPath: /etc/secrets
+ {{- end }}
+ - name: config
+ mountPath: /etc/flyte/config.d
+ - name: webhook-certs
+ mountPath: /var/run/flyte/certs
+ readOnly: false
+ {{- if .Values.deployment.extraVolumeMounts }}
+ {{- tpl ( .Values.deployment.extraVolumeMounts | toYaml ) . | nindent 12 }}
+ {{- end }}
+ {{- if .Values.deployment.securityContext }}
+ securityContext: {{- toYaml .Values.deployment.securityContext | nindent 12 }}
+ {{- end }}
+ {{- if .Values.deployment.sidecars }}
+ {{- tpl ( .Values.deployment.sidecars | toYaml ) . | nindent 8 }}
+ {{- end }}
+ volumes:
+ - name: config
+ {{- if (include "flyte-binary.configuration.externalConfiguration" .) }}
+ projected:
+ sources:
+ {{- if .Values.configuration.externalConfigMap }}
+ - configMap:
+ name: {{ tpl .Values.configuration.externalConfigMap . }}
+ {{- end }}
+ {{- if .Values.configuration.externalSecretRef }}
+ - secret:
+ name: {{ tpl .Values.configuration.externalSecretRef . }}
+ {{- end }}
+ {{- else }}
+ projected:
+ sources:
+ - configMap:
+ name: {{ include "flyte-binary.configuration.configMapName" . }}
+ - secret:
+ name: {{ include "flyte-binary.configuration.configSecretName" . }}
+ {{- if .Values.configuration.inlineConfigMap }}
+ - configMap:
+ name: {{ tpl .Values.configuration.inlineConfigMap . }}
+ {{- end }}
+ {{- if .Values.configuration.inlineSecretRef }}
+ - secret:
+ name: {{ tpl .Values.configuration.inlineSecretRef . }}
+ {{- end }}
+ {{- end }}
+ - name: webhook-certs
+ emptyDir: {}
+ {{- if .Values.deployment.extraVolumes }}
+ {{- tpl ( .Values.deployment.extraVolumes | toYaml ) . | nindent 8 }}
+ {{- end }}
diff --git a/charts/flyte-binary/templates/ingress/grpc.yaml b/charts/flyte-binary/templates/ingress/grpc.yaml
new file mode 100644
index 00000000000..bd43185e509
--- /dev/null
+++ b/charts/flyte-binary/templates/ingress/grpc.yaml
@@ -0,0 +1,64 @@
+{{- if and .Values.ingress.create .Values.ingress.separateGrpcIngress }}
+{{- $paths := (include "flyte-binary.ingress.grpcPaths" .) | fromYamlArray }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ name: {{ include "flyte-binary.fullname" . }}-grpc
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-binary.labels" . | nindent 4 }}
+ {{- if .Values.commonLabels }}
+ {{- tpl ( .Values.commonLabels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.ingress.labels }}
+ {{- tpl ( .Values.ingress.labels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- if .Values.commonAnnotations }}
+ {{- tpl ( .Values.commonAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.ingress.commonAnnotations }}
+ {{- tpl ( .Values.ingress.commonAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.ingress.grpcAnnotations }}
+ {{- tpl ( .Values.ingress.grpcAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+spec:
+ {{- if .Values.ingress.grpcIngressClassName }}
+ ingressClassName: {{ .Values.ingress.grpcIngressClassName | quote }}
+ {{- else if .Values.ingress.ingressClassName }}
+ ingressClassName: {{ .Values.ingress.ingressClassName | quote }}
+ {{- end }}
+ {{- if .Values.ingress.grpcTls }}
+ tls: {{- tpl ( .Values.ingress.grpcTls | toYaml ) . | nindent 2 }}
+ {{- else if .Values.ingress.tls }}
+ tls: {{- tpl ( .Values.ingress.tls | toYaml ) . | nindent 2 }}
+ {{- end }}
+ rules:
+ - http:
+ paths:
+ {{- if .Values.ingress.grpcExtraPaths.prepend }}
+ {{- tpl ( .Values.ingress.grpcExtraPaths.prepend | toYaml ) . | nindent 6 }}
+ {{- end }}
+ {{- range $path := $paths }}
+ - path: {{ $path }}
+ {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }}
+ pathType: ImplementationSpecific
+ {{- end }}
+ backend:
+ {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+ service:
+ name: {{ include "flyte-binary.service.grpc.name" $ }}
+ port:
+ number: {{ include "flyte-binary.service.grpc.port" $ }}
+ {{- else }}
+ serviceName: {{ include "flyte-binary.service.grpc.name" $ }}
+ servicePort: {{ include "flyte-binary.service.grpc.port" $ }}
+ {{- end }}
+ {{- end }}
+ {{- if .Values.ingress.grpcExtraPaths.append }}
+ {{- tpl ( .Values.ingress.grpcExtraPaths.append | toYaml ) . | nindent 6 }}
+ {{- end }}
+ {{- if .Values.ingress.host }}
+ host: {{ tpl .Values.ingress.host . | quote }}
+ {{- end }}
+{{- end }}
diff --git a/charts/flyte-binary/templates/ingress/http.yaml b/charts/flyte-binary/templates/ingress/http.yaml
new file mode 100644
index 00000000000..cdce87047bd
--- /dev/null
+++ b/charts/flyte-binary/templates/ingress/http.yaml
@@ -0,0 +1,199 @@
+{{- if .Values.ingress.create }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ name: {{ include "flyte-binary.fullname" . }}-http
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-binary.labels" . | nindent 4 }}
+ {{- if .Values.commonLabels }}
+ {{- tpl ( .Values.commonLabels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.ingress.labels }}
+ {{- tpl ( .Values.ingress.labels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- if .Values.commonAnnotations }}
+ {{- tpl ( .Values.commonAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.ingress.commonAnnotations }}
+ {{- tpl ( .Values.ingress.commonAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.ingress.httpAnnotations }}
+ {{- tpl ( .Values.ingress.httpAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+spec:
+ {{- if .Values.ingress.httpIngressClassName }}
+ ingressClassName: {{ .Values.ingress.httpIngressClassName | quote }}
+ {{- else if .Values.ingress.ingressClassName }}
+ ingressClassName: {{ .Values.ingress.ingressClassName | quote }}
+ {{- end }}
+ {{- if .Values.ingress.httpTls }}
+ tls: {{- tpl ( .Values.ingress.httpTls | toYaml ) . | nindent 2 }}
+ {{- else if .Values.ingress.tls }}
+ tls: {{- tpl ( .Values.ingress.tls | toYaml ) . | nindent 2 }}
+ {{- end }}
+ rules:
+ - http:
+ paths:
+ {{- if .Values.ingress.httpExtraPaths.prepend }}
+ {{- tpl ( .Values.ingress.httpExtraPaths.prepend | toYaml ) . | nindent 6 }}
+ {{- end }}
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /console
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /console/*
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /api
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /api/*
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /healthcheck
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /v1/*
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /.well-known
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /.well-known/*
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /login
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /login/*
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /logout
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /logout/*
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /callback
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /callback/*
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /me
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /config
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /config/*
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /oauth2
+ pathType: ImplementationSpecific
+ - backend:
+ service:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ port:
+ number: {{ include "flyte-binary.service.http.port" . }}
+ path: /oauth2/*
+ pathType: ImplementationSpecific
+ {{- if not .Values.ingress.separateGrpcIngress }}
+ {{- $paths := (include "flyte-binary.ingress.grpcPaths" .) | fromYamlArray }}
+ {{- range $path := $paths }}
+ - path: {{ $path }}
+ {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }}
+ pathType: ImplementationSpecific
+ {{- end }}
+ backend:
+ {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+ service:
+ name: {{ include "flyte-binary.service.http.name" $ }}
+ port:
+ number: {{ include "flyte-binary.service.grpc.port" $ }}
+ {{- else }}
+ serviceName: {{ include "flyte-binary.service.http.name" $ }}
+ servicePort: {{ include "flyte-binary.service.grpc.port" $ }}
+ {{- end }}
+ {{- end }}
+ {{- end }}
+ {{- if .Values.ingress.httpExtraPaths.append }}
+ {{- tpl ( .Values.ingress.httpExtraPaths.append | toYaml ) . | nindent 6 }}
+ {{- end }}
+ {{- if .Values.ingress.host }}
+ host: {{ tpl .Values.ingress.host . | quote }}
+ {{- end }}
+{{- end }}
diff --git a/charts/flyte-binary/templates/service/http.yaml b/charts/flyte-binary/templates/service/http.yaml
new file mode 100644
index 00000000000..ab79c90210d
--- /dev/null
+++ b/charts/flyte-binary/templates/service/http.yaml
@@ -0,0 +1,59 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "flyte-binary.service.http.name" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-binary.labels" . | nindent 4 }}
+ {{- if .Values.commonLabels }}
+ {{- tpl ( .Values.commonLabels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.service.labels }}
+ {{- tpl ( .Values.service.labels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- if .Values.commonAnnotations }}
+ {{- tpl ( .Values.commonAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.service.commonAnnotations }}
+ {{- tpl ( .Values.service.commonAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.service.httpAnnotations }}
+ {{- tpl ( .Values.service.httpAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+spec:
+ type: {{ .Values.service.type }}
+ {{- if or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort") }}
+ externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy | quote }}
+ {{- end }}
+ {{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerSourceRanges)) }}
+ loadBalancerSourceRanges: {{ .Values.service.loadBalancerSourceRanges }}
+ {{- end }}
+ {{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerIP)) }}
+ loadBalancerIP: {{ .Values.service.loadBalancerIP }}
+ {{- end }}
+ {{- if and .Values.service.clusterIP (eq .Values.service.type "ClusterIP") }}
+ clusterIP: {{ .Values.service.clusterIP }}
+ {{- end }}
+ ports:
+ - name: http
+ port: {{ include "flyte-binary.service.http.port" . }}
+ targetPort: http
+ {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.http)) }}
+ nodePort: {{ .Values.service.nodePorts.http }}
+ {{- else if eq .Values.service.type "ClusterIP" }}
+ nodePort: null
+ {{- end }}
+ {{- if not .Values.ingress.separateGrpcIngress }}
+ - name: grpc
+ port: {{ include "flyte-binary.service.grpc.port" . }}
+ targetPort: grpc
+ {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.grpc)) }}
+ nodePort: {{ .Values.service.nodePorts.grpc }}
+ {{- else if eq .Values.service.type "ClusterIP" }}
+ nodePort: null
+ {{- end }}
+ {{- end }}
+ {{- if .Values.service.extraPorts }}
+ {{- tpl ( .Values.service.extraPorts | toYaml ) . | nindent 4 }}
+ {{- end }}
+ selector: {{- include "flyte-binary.selectorLabels" . | nindent 4 }}
diff --git a/charts/flyte-binary/templates/service/webhook.yaml b/charts/flyte-binary/templates/service/webhook.yaml
new file mode 100644
index 00000000000..0fa6d45845c
--- /dev/null
+++ b/charts/flyte-binary/templates/service/webhook.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "flyte-binary.webhook.serviceName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-binary.labels" . | nindent 4 }}
+spec:
+ selector: {{- include "flyte-binary.selectorLabels" . | nindent 4 }}
+ ports:
+ - name: webhook
+ port: 443
+ targetPort: 9443
+ protocol: TCP
diff --git a/charts/flyte-binary/templates/serviceaccount.yaml b/charts/flyte-binary/templates/serviceaccount.yaml
new file mode 100644
index 00000000000..52c077e4eb6
--- /dev/null
+++ b/charts/flyte-binary/templates/serviceaccount.yaml
@@ -0,0 +1,25 @@
+{{- if .Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ include "flyte-binary.serviceAccountName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-binary.labels" . | nindent 4 }}
+ {{- if .Values.commonLabels }}
+ {{- tpl ( .Values.commonLabels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.serviceAccount.labels }}
+ {{- tpl ( .Values.serviceAccount.labels | toYaml ) . | nindent 4 }}
+ {{- end }}
+ annotations:
+ {{- if .Values.commonAnnotations }}
+ {{- tpl ( .Values.commonAnnotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+ {{- if .Values.serviceAccount.annotations }}
+ {{- tpl ( .Values.serviceAccount.annotations | toYaml ) . | nindent 4 }}
+ {{- end }}
+{{- if .Values.serviceAccount.imagePullSecrets }}
+imagePullSecrets:
+ {{- tpl ( .Values.serviceAccount.imagePullSecrets | toYaml ) . | nindent 2 }}
+{{- end }}
+{{- end }}
diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml
new file mode 100644
index 00000000000..a9e86177237
--- /dev/null
+++ b/charts/flyte-binary/values.yaml
@@ -0,0 +1,435 @@
+# nameOverride String to override flyte-binary.name template
+nameOverride: ""
+# fullnameOverride String to override flyte-binary.fullname template
+fullnameOverride: ""
+# commonLabels Add labels to all the deployed resources
+commonLabels: {}
+# commonAnnotations Add annotations to all the deployed resources
+commonAnnotations: {}
+
+# flyte-core-components: Configuration for the unified Flyte Manager
+flyte-core-components:
+ manager:
+ server:
+ host: "0.0.0.0"
+ port: 8090
+ executor:
+ healthProbePort: 8081
+ kubernetes:
+ namespace: "flyte"
+ kubeconfig: ""
+ qps: 1000
+ burst: 2000
+ timeout: "30s"
+ runs:
+ server:
+ host: "0.0.0.0"
+ port: 8090
+ watchBufferSize: 100
+ storagePrefix: "s3://flyte-data"
+ # Right now sandbox use SQLite db, we just leave the postgreSQL setting for potential future usage
+ database:
+ maxIdleConnections: 10
+ maxOpenConnections: 100
+ connMaxLifeTime: 1h
+ postgres:
+ host: "127.0.0.1"
+ port: 5432
+ dbname: flyte
+ username: postgres
+ password: ""
+ options: "sslmode=disable"
+ debug: false
+
+ actions:
+ kubernetes:
+ namespace: "flyte"
+ kubeconfig: ""
+ watchBufferSize: 100
+
+ dataproxy:
+ upload:
+ maxSize: "100Mi"
+ maxExpiresIn: 1h
+ defaultFileNameLength: 20
+ storagePrefix: "uploads"
+ download:
+ maxExpiresIn: 1h
+
+ secret:
+ kubernetes:
+ namespace: "flyte"
+ clusterName: "flyte-devbox"
+ kubeconfig: ""
+ qps: 100
+ burst: 200
+ timeout: "30s"
+
+# configuration Specify configuration for Flyte
+configuration:
+ # database Specify configuration for Flyte's database connection
+ database:
+ # username Name for user to connect to database as
+ username: postgres
+ # password Password to connect to database with
+ # If set, a Secret will be created with this value and mounted to Flyte pod
+ password: ""
+ # passwordPath Mountpath of file containing password to be added to Flyte deployment
+ passwordPath: ""
+ # host Hostname of database instance
+ host: 127.0.0.1
+ # port Port to connect to database at
+ port: 5432
+ # dbname Name of database to use
+ dbname: flyte
+ # options Additional client options for connecting to database
+ options: sslmode=disable
+ # storage Specify configuration for object store
+ storage:
+ # metadataContainer Bucket to store Flyte metadata
+ metadataContainer: "my-organization-flyte-container"
+ # userDataContainer Bucket to store Flyte user data
+ userDataContainer: "my-organization-flyte-container"
+ # provider Object store provider (Supported values: s3, gcs)
+ provider: s3
+ # providerConfig Additional object store provider-specific configuration
+ providerConfig:
+ # s3 Provider configuration for S3 object store
+ s3:
+ # region AWS region at which bucket resides
+ region: "us-east-1"
+ # disableSSL Switch to disable SSL for communicating with S3-compatible service
+ disableSSL: false
+ # v2Signing Flag to sign requests with v2 signature
+ # Useful for s3-compatible blob stores (e.g. minio)
+ v2Signing: false
+ # endpoint URL of S3-compatible service
+ endpoint: ""
+ # authType Type of authentication to use for connecting to S3-compatible service (Supported values: iam, accesskey)
+ authType: "iam"
+ # accessKey Access key for authenticating with S3-compatible service
+ accessKey: ""
+ # secretKey Secret key for authenticating with S3-compatible service
+ secretKey: ""
+ # gcs Provider configuration for GCS object store
+ gcs:
+ # project Google Cloud project in which bucket resides
+ project: "my-organization-gcp-project"
+ # azure Provider configuration for Azure object store
+ azure:
+ # configDomainSuffix Domain name suffix
+ configDomainSuffix: ""
+ # configUploadConcurrency Upload Concurrency (default 4)
+ configUploadConcurrency: 4
+ # account Storage Account name
+ account: "storage-account-name"
+ # key Storage Account key if used
+ key: ""
+ # logging Specify configuration for logs emitted by Flyte
+ logging:
+ # level Set the log level
+ level: 1
+ # plugins Specify additional logging plugins
+ plugins:
+ # kubernetes Configure logging plugin to have logs visible in the Kubernetes Dashboard
+ kubernetes:
+ enabled: false
+ templateUri: ""
+ # cloudwatch Configure logging plugin to have logs visible in CloudWatch
+ cloudwatch:
+ enabled: false
+ templateUri: ""
+ # stackdriver Configure logging plugin to have logs visible in StackDriver
+ stackdriver:
+ enabled: false
+ templateUri: ""
+ custom: []
+ # auth Specify configuration for Flyte authentication
+ # Todo(alex): When we have a auth service, we will come back here to set configuration
+ auth:
+ # enabled Enable Flyte authentication
+ enabled: false
+ # enableAuthServer Enable built-in authentication server
+ enableAuthServer: true
+ # oidc OIDC configuration for Flyte authentication
+ oidc:
+ # baseUrl URL for OIDC provider
+ baseUrl: ""
+ # clientId Flyte application client ID
+ clientId: ""
+ # clientSecret Flyte application client secret
+ clientSecret: ""
+ # internal Configuration for internal authentication
+ # The settings for internal still need to be defined if you wish to use an external auth server
+ # These credentials are used during communication between the FlyteAdmin and Propeller microservices
+ internal:
+ # clientId Client ID for internal authentication - set to external auth server
+ clientId: flytemanager
+ # clientSecret Client secret for internal authentication
+ clientSecret: ""
+ # clientSecretHash Bcrypt hash of clientSecret
+ clientSecretHash: ""
+ # Uncomment next line if needed - set this field if your external Auth server (ex. Auth0) requires an audience parameter
+ # audience: ""
+ # flyteClient Configuration for Flyte client authentication
+ flyteClient:
+ # clientId Client ID for Flyte client authentication
+ clientId: flytectl
+ # redirectUri Redirect URI for Flyte client authentication
+ redirectUri: http://localhost:53593/callback
+ # scopes Scopes for Flyte client authentication
+ scopes:
+ - all
+ # audience Audience for Flyte client authentication
+ audience: ""
+ # authorizedUris Set of URIs that clients are allowed to visit the service on
+ authorizedUris: []
+ # clientSecretExternalSecretRef Specify an existing, external Secret containing values for `client_secret` and `oidc_client_secret`.
+ # If set, a Secret will not be generated by this chart for client secrets.
+ clientSecretsExternalSecretRef: ""
+ # co-pilot Configuration for Flyte CoPilot
+ co-pilot:
+ # image Configure image to use for CoPilot sidecar
+ image:
+ # repository CoPilot sidecar image repository
+ repository: cr.flyte.org/flyteorg/flytecopilot # FLYTECOPILOT_IMAGE
+ # tag CoPilot sidecar image tag
+ tag: v1.16.4 # FLYTECOPILOT_TAG
+ # connectorService Flyte Connector configuration
+ connectorService:
+ defaultConnector:
+ endpoint: "k8s://flyteconnector.flyte:8000"
+ insecure: true
+ timeouts:
+ GetTask: 10s
+ ListAgents: 3s
+ defaultTimeout: 10s
+
+ # externalConfigMap Specify an existing, external ConfigMap to use as configuration for Flyte
+ # If set, no Flyte configuration will be generated by this chart
+ externalConfigMap: ""
+ # externalSecretRef Specify an existing, external Secret to use as configuration for Flyte
+ # If set, no Flyte configuration will be generated by this chart
+ externalSecretRef: ""
+ # inline Specify additional configuration or overrides for Flyte, to be merged with the base configuration
+ inline: {}
+ # inlineConfigMap Specify an existing ConfigMap containing additional configuration
+ # or overrides for Flyte, to be merged with the base configuration
+ inlineConfigMap: ""
+ # inlineSecretRef Specify an existing Secret containing additional configuration
+ # or overrides for Flyte, to be merged with the base configuration
+ inlineSecretRef: ""
+ # labels Add labels to created ConfigMap
+ labels: {}
+ # annotations Add annotations to created ConfigMap
+ annotations: {}
+
+# deployment Configure Flyte deployment specification
+deployment:
+ # image Configure image to use for Flyte
+ image:
+ # repository Flyte image repository
+ repository: cr.flyte.org/flyteorg/flyte-binary # FLYTE_IMAGE
+ # tag Flyte image tag
+ tag: latest # FLYTE_TAG
+ # pullPolicy Flyte image pull policy
+ pullPolicy: IfNotPresent
+ # extraEnvVars Array with extra environment variables to add to Flyte
+ extraEnvVars: []
+ # extraEnvVarsConfigMap Name of existing ConfigMap containing extra env vars for Flyte
+ extraEnvVarsConfigMap: ""
+ # extraEnvVarsSecret Name of existing Secret containing extra env vars for Flyte
+ extraEnvVarsSecret: ""
+ # command Override default container command
+ command: []
+ # args Override default container args
+ args: []
+ # livenessProbe Override default container liveness probe
+ # See: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
+ livenessProbe: {}
+ # readinessProbe Override default container readiness probe
+ readinessProbe: {}
+ # startupProbe Specify a startup probe for Flyte container
+ startupProbe: {}
+ # lifecycleHooks Specify hooks to run in Flyte container before or after startup
+ lifecycleHooks: {}
+ # resources Resource limits and requests for Flyte container
+ # Uncomment and update to specify resources for deployment
+ # resources:
+ # limits:
+ # memory: 1Gi
+ # requests:
+ # cpu: 1
+ # podSecurityContext Specify security context for Flyte pod
+ # See: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+ podSecurityContext:
+ enabled: false
+ runAsUser: 65534
+ runAsGroup: 65534
+ fsGroup: 65534
+ # waitForDB Configure init container to wait for DB during pod startup
+ # This is disabled when an external ConfigMap is used to specify Flyte configuration
+ waitForDB:
+ # image Configure image to use for wait-for-db init container
+ image:
+ # repository Init container image repository
+ repository: postgres
+ # tag Init container image tag
+ tag: 15-alpine
+ # pullPolicy Init container image pull policy
+ pullPolicy: IfNotPresent
+ # command Override default init container command
+ command: []
+ # args Override default init container args
+ args: []
+ # securityContext Specify security context for wait-for-db init container
+ securityContext: {}
+ # genAdminAuthSecret Configure init container to generate secrets for internal use
+ genAdminAuthSecret:
+ # command Override default init container command
+ command: []
+ # args Override default init container args
+ args: []
+ # securityContext Specify security context for gen-admin-auth-secret init container
+ securityContext: {}
+ # labels Add labels to Flyte deployment
+ labels: {}
+ # annotations Add annotations to Flyte deployment
+ annotations: {}
+ # labels Add labels to Flyte pod
+ podLabels: {}
+ # annotations Add annotations to Flyte pod
+ podAnnotations: {}
+ # extraVolumeMounts Specify additional volumeMounts for Flyte container
+ extraVolumeMounts: []
+ # extraVolume Specify additional volumes for Flyte pod
+ extraVolumes: []
+ # sidecars Specify additional containers for Flyte pod
+ sidecars: []
+ # initContainers Specify additional init containers for Flyte pod
+ initContainers: []
+ # preInitContainers Specify additional pre-init containers for Flyte pod that run before the wait-for-db init container
+ preInitContainers: []
+ # extraPodSpec Specify additional configuration for Flyte pod
+ # This can be used for adding affinity, tolerations, hostNetwork, etc.
+ extraPodSpec: {}
+ # securityContext Specify security context for Flyte container
+ securityContext: {}
+
+# service Configure service for Flyte
+service:
+ # type Kubernetes service type
+ type: ClusterIP
+ # ports Flyte service ports
+ # If not specified, defaults to corresponding container ports
+ ports:
+ http: ""
+ grpc: ""
+ # nodePorts Node ports for Flyte service if service type is `NodePort` or `LoadBalancer`
+ nodePorts:
+ http: ""
+ grpc: ""
+ # clusterIP Set static IP if service type is `ClusterIP`
+ clusterIP: ""
+ # labels Add labels to Flyte services
+ labels: {}
+ # commonAnnotations Add annotations to Flyte services
+ commonAnnotations: {}
+ # httpAnnotations Add annotations to http service resource
+ httpAnnotations: {}
+ # grpcAnnotations Add annotations to grpc service resource
+ grpcAnnotations: {}
+ # loadBalancerIP Set static IP if service type is `LoadBalancer`
+ loadBalancerIP: ""
+ # externalTrafficPolicy Enable client source IP preservation if service type is `NodePort` or `LoadBalancer`
+ # See: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip
+ externalTrafficPolicy: Cluster
+ # loadBalancerSourceRanges Addresses that are allowed when service is `LoadBalancer`
+ loadBalancerSourceRanges: []
+ # extraPorts Additional ports to add to Flyte service
+ extraPorts: []
+
+# ingress Configure ingress for Flyte
+ingress:
+ # create Create ingress resources
+ create: false
+ # labels Add labels to ingress resources
+ labels: {}
+ # host Hostname to bind to ingress resources
+ host: ""
+ # separateGrpcIngress Create a separate ingress resource for GRPC if true. Required for certain ingress controllers like nginx.
+ separateGrpcIngress: true
+ # commonAnnotations Add common annotations to all ingress resources
+ commonAnnotations: {}
+ # httpAnnotations Add annotations to http ingress resource
+ httpAnnotations: {}
+ # grpcAnnotations Add annotations to grpc ingress resource
+ grpcAnnotations: {}
+ # ingressClassName Ingress class to use with all ingress resources
+ ingressClassName: ""
+ # httpIngressClassName Ingress class to use with all http ingress resource. Overrides `ingressClassName`
+ httpIngressClassName: ""
+ # grpcIngressClassName Ingress class to use with all grpc ingress resource. Overrides `ingressClassName`
+ grpcIngressClassName: ""
+ # tls Add TLS configuration to all ingress resources
+ tls: []
+ # httpTls Add TLS configuration to http ingress resource. Overrides `tls`
+ httpTls: []
+ # grpcTls Add TLS configuration to grpc ingress resource. Overrides `tls`
+ grpcTls: []
+ # httpExtraPaths Add extra paths to http ingress rule
+ httpExtraPaths:
+ prepend: []
+ append: []
+ # grpcExtraPaths Add extra paths to grpc ingress rule
+ grpcExtraPaths:
+ prepend: []
+ append: []
+
+# rbac Configure Kubernetes RBAC for Flyte
+rbac:
+ # create Create ClusterRole and ClusterRoleBinding resources
+ create: true
+ # labels Add labels to RBAC resources
+ labels: {}
+ # annotations Add annotations to RBAC resources
+ annotations: {}
+ # extraRules Add additional rules to the ClusterRole
+ extraRules: []
+
+# serviceAccount Configure Flyte ServiceAccount
+serviceAccount:
+ # create Create ServiceAccount for Flyte
+ create: true
+ # name Name of service account
+ name: ""
+ # labels Add labels to ServiceAccount
+ labels: {}
+ # annotations Add annotations to ServiceAccount
+ annotations: {}
+ # imagePullSecrets Secrets to use for fetching images from private registries
+ imagePullSecrets: []
+
+# flyteconnector Configure Flyte Connector objects
+flyteconnector:
+ # enable Flag to enable bundled Flyte Connector
+ enabled: false
+
+enabled_plugins:
+ # -- Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig)
+ tasks:
+ # -- Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig)
+ task-plugins:
+ # -- [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config).
+ # Enable sagemaker*, athena if you install the backend plugins
+ enabled-plugins:
+ - container
+ - sidecar
+ - connector-service
+ - echo
+ default-for-task-types:
+ container: container
+ sidecar: sidecar
+ container_array: k8s-array
+ # -- Uncomment to enable task type that uses Flyte Connector
+ # bigquery_query_job_task: connector-service
diff --git a/charts/flyte-devbox/.helmignore b/charts/flyte-devbox/.helmignore
new file mode 100644
index 00000000000..0e8a0eb36f4
--- /dev/null
+++ b/charts/flyte-devbox/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/charts/flyte-devbox/Chart.lock b/charts/flyte-devbox/Chart.lock
new file mode 100644
index 00000000000..551f4c2f2f6
--- /dev/null
+++ b/charts/flyte-devbox/Chart.lock
@@ -0,0 +1,10 @@
+dependencies:
+- name: docker-registry
+ repository: https://twuni.github.io/docker-registry.helm
+ version: 2.2.2
+- name: flyte-binary
+ repository: file://../flyte-binary
+ version: v0.2.0
+
+digest: sha256:bc1e2e37ac2f11f9bf547262643c9076d6c86af661b6ffdee77cc6756d687d6e
+generated: "2026-04-14T10:42:50.772509+08:00"
diff --git a/charts/flyte-devbox/Chart.yaml b/charts/flyte-devbox/Chart.yaml
new file mode 100644
index 00000000000..262b044a79f
--- /dev/null
+++ b/charts/flyte-devbox/Chart.yaml
@@ -0,0 +1,34 @@
+apiVersion: v2
+name: flyte-devbox
+description: A Helm chart for the Flyte local demo cluster
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.1"
+
+dependencies:
+ - name: docker-registry
+ version: 2.2.2
+ repository: https://twuni.github.io/docker-registry.helm
+ condition: docker-registry.enabled
+ - name: flyte-binary
+ version: v0.2.0
+ repository: file://../flyte-binary
+ condition: flyte-binary.enabled
diff --git a/charts/flyte-devbox/README.md b/charts/flyte-devbox/README.md
new file mode 100644
index 00000000000..d7aa6571603
--- /dev/null
+++ b/charts/flyte-devbox/README.md
@@ -0,0 +1,115 @@
+# flyte-devbox
+
+  
+
+A Helm chart for the Flyte local demo cluster
+
+## Requirements
+
+| Repository | Name | Version |
+|------------|------|---------|
+| file://../flyte-binary | flyte-binary | v0.2.0 |
+| https://charts.bitnami.com/bitnami | minio | 12.6.7 |
+| https://twuni.github.io/docker-registry.helm | docker-registry | 2.2.2 |
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| docker-registry.enabled | bool | `true` | |
+| docker-registry.fullnameOverride | string | `"docker-registry"` | |
+| docker-registry.image.pullPolicy | string | `"Never"` | |
+| docker-registry.image.tag | string | `"sandbox"` | |
+| docker-registry.persistence.enabled | bool | `false` | |
+| docker-registry.secrets.haSharedSecret | string | `"flytesandboxsecret"` | |
+| docker-registry.service.nodePort | int | `30000` | |
+| docker-registry.service.type | string | `"NodePort"` | |
+| flyte-binary.clusterResourceTemplates.inlineConfigMap | string | `"{{ include \"flyte-devbox.clusterResourceTemplates.inlineConfigMap\" . }}"` | |
+| flyte-binary.configuration.database.host | string | `"postgresql"` | |
+| flyte-binary.configuration.database.password | string | `"postgres"` | |
+| flyte-binary.configuration.inline.plugins.k8s.default-env-vars[0].FLYTE_AWS_ENDPOINT | string | `"http://minio.{{ .Release.Namespace }}:9000"` | |
+| flyte-binary.configuration.inline.plugins.k8s.default-env-vars[1].FLYTE_AWS_ACCESS_KEY_ID | string | `"minio"` | |
+| flyte-binary.configuration.inline.plugins.k8s.default-env-vars[2].FLYTE_AWS_SECRET_ACCESS_KEY | string | `"miniostorage"` | |
+| flyte-binary.configuration.inline.plugins.k8s.default-env-vars[3]._U_EP_OVERRIDE | string | `"flyte-binary-http.{{ .Release.Namespace }}:8090"` | |
+| flyte-binary.configuration.inline.plugins.k8s.default-env-vars[4]._U_INSECURE | string | `"true"` | |
+| flyte-binary.configuration.inline.plugins.k8s.default-env-vars[5]._U_USE_ACTIONS | string | `"1"` | |
+| flyte-binary.configuration.inline.runs.database.postgres.dbName | string | `"runs"` | |
+| flyte-binary.configuration.inline.runs.database.postgres.host | string | `"postgresql.{{ .Release.Namespace }}"` | |
+| flyte-binary.configuration.inline.runs.database.postgres.password | string | `"postgres"` | |
+| flyte-binary.configuration.inline.runs.database.postgres.port | int | `5432` | |
+| flyte-binary.configuration.inline.runs.database.postgres.user | string | `"postgres"` | |
+| flyte-binary.configuration.inline.storage.signedURL.stowConfigOverride.endpoint | string | `"http://localhost:30002"` | |
+| flyte-binary.configuration.inline.task_resources.defaults.cpu | string | `"500m"` | |
+| flyte-binary.configuration.inline.task_resources.defaults.ephemeralStorage | int | `0` | |
+| flyte-binary.configuration.inline.task_resources.defaults.gpu | int | `0` | |
+| flyte-binary.configuration.inline.task_resources.defaults.memory | string | `"1Gi"` | |
+| flyte-binary.configuration.inline.task_resources.limits.cpu | int | `0` | |
+| flyte-binary.configuration.inline.task_resources.limits.ephemeralStorage | int | `0` | |
+| flyte-binary.configuration.inline.task_resources.limits.gpu | int | `0` | |
+| flyte-binary.configuration.inline.task_resources.limits.memory | int | `0` | |
+| flyte-binary.configuration.inlineConfigMap | string | `"{{ include \"flyte-devbox.configuration.inlineConfigMap\" . }}"` | |
+| flyte-binary.configuration.logging.level | int | `5` | |
+| flyte-binary.configuration.storage.metadataContainer | string | `"flyte-data"` | |
+| flyte-binary.configuration.storage.provider | string | `"s3"` | |
+| flyte-binary.configuration.storage.providerConfig.s3.accessKey | string | `"minio"` | |
+| flyte-binary.configuration.storage.providerConfig.s3.authType | string | `"accesskey"` | |
+| flyte-binary.configuration.storage.providerConfig.s3.disableSSL | bool | `true` | |
+| flyte-binary.configuration.storage.providerConfig.s3.endpoint | string | `"http://minio.{{ .Release.Namespace }}:9000"` | |
+| flyte-binary.configuration.storage.providerConfig.s3.secretKey | string | `"miniostorage"` | |
+| flyte-binary.configuration.storage.providerConfig.s3.v2Signing | bool | `true` | |
+| flyte-binary.configuration.storage.userDataContainer | string | `"flyte-data"` | |
+| flyte-binary.deployment.image.pullPolicy | string | `"Never"` | |
+| flyte-binary.deployment.image.repository | string | `"flyte-binary-v2"` | |
+| flyte-binary.deployment.image.tag | string | `"sandbox"` | |
+| flyte-binary.deployment.livenessProbe.httpGet.path | string | `"/healthz"` | |
+| flyte-binary.deployment.livenessProbe.httpGet.port | string | `"http"` | |
+| flyte-binary.deployment.readinessProbe.httpGet.path | string | `"/readyz"` | |
+| flyte-binary.deployment.readinessProbe.httpGet.port | string | `"http"` | |
+| flyte-binary.deployment.readinessProbe.periodSeconds | int | `1` | |
+| flyte-binary.deployment.startupProbe.failureThreshold | int | `30` | |
+| flyte-binary.deployment.startupProbe.httpGet.path | string | `"/healthz"` | |
+| flyte-binary.deployment.startupProbe.httpGet.port | string | `"http"` | |
+| flyte-binary.deployment.startupProbe.periodSeconds | int | `1` | |
+| flyte-binary.deployment.waitForDB.image.pullPolicy | string | `"Never"` | |
+| flyte-binary.deployment.waitForDB.image.repository | string | `"bitnami/postgresql"` | |
+| flyte-binary.deployment.waitForDB.image.tag | string | `"sandbox"` | |
+| flyte-binary.enabled | bool | `true` | |
+| flyte-binary.fullnameOverride | string | `"flyte-binary"` | |
+| flyte-binary.rbac.extraRules[0].apiGroups[0] | string | `"*"` | |
+| flyte-binary.rbac.extraRules[0].resources[0] | string | `"*"` | |
+| flyte-binary.rbac.extraRules[0].verbs[0] | string | `"*"` | |
+| minio.auth.rootPassword | string | `"miniostorage"` | |
+| minio.auth.rootUser | string | `"minio"` | |
+| minio.defaultBuckets | string | `"flyte-data"` | |
+| minio.enabled | bool | `true` | |
+| minio.extraEnvVars[0].name | string | `"MINIO_BROWSER_REDIRECT_URL"` | |
+| minio.extraEnvVars[0].value | string | `"http://localhost:30080/minio"` | |
+| minio.fullnameOverride | string | `"minio"` | |
+| minio.image.pullPolicy | string | `"Never"` | |
+| minio.image.tag | string | `"sandbox"` | |
+| minio.persistence.enabled | bool | `true` | |
+| minio.persistence.existingClaim | string | `"{{ include \"flyte-devbox.persistence.minioVolumeName\" . }}"` | |
+| minio.service.nodePorts.api | int | `30002` | |
+| minio.service.type | string | `"NodePort"` | |
+| minio.volumePermissions.enabled | bool | `true` | |
+| minio.volumePermissions.image.pullPolicy | string | `"Never"` | |
+| minio.volumePermissions.image.tag | string | `"sandbox"` | |
+| postgresql.auth.postgresPassword | string | `"postgres"` | |
+| postgresql.enabled | bool | `true` | |
+| postgresql.fullnameOverride | string | `"postgresql"` | |
+| postgresql.image.pullPolicy | string | `"Never"` | |
+| postgresql.image.tag | string | `"sandbox"` | |
+| postgresql.primary.persistence.enabled | bool | `true` | |
+| postgresql.primary.persistence.existingClaim | string | `"{{ include \"flyte-devbox.persistence.dbVolumeName\" . }}"` | |
+| postgresql.primary.service.nodePorts.postgresql | int | `30001` | |
+| postgresql.primary.service.type | string | `"NodePort"` | |
+| postgresql.shmVolume.enabled | bool | `false` | |
+| postgresql.volumePermissions.enabled | bool | `true` | |
+| postgresql.volumePermissions.image.pullPolicy | string | `"Never"` | |
+| postgresql.volumePermissions.image.tag | string | `"sandbox"` | |
+| sandbox.console.enabled | bool | `true` | |
+| sandbox.console.image.pullPolicy | string | `"Never"` | |
+| sandbox.console.image.repository | string | `"ghcr.io/flyteorg/flyte-client-v2"` | |
+| sandbox.console.image.tag | string | `"latest"` | |
+| sandbox.dev | bool | `false` | |
+
diff --git a/charts/flyte-devbox/charts/flyte-binary/Makefile b/charts/flyte-devbox/charts/flyte-binary/Makefile
new file mode 100644
index 00000000000..2f175d4c468
--- /dev/null
+++ b/charts/flyte-devbox/charts/flyte-binary/Makefile
@@ -0,0 +1,17 @@
+
+GENERATED_FILE="gen/binary_manifest.yaml"
+DEVBOX_GENERATED_FILE="gen/devboxmanifest.yaml"
+
+cleanhelm:
+ @[ -f $(GENERATED_FILE) ] && rm $(GENERATED_FILE) || true
+
+cleandevbox:
+ @[ -f $(DEVBOX_GENERATED_FILE) ] && rm $(DEVBOX_GENERATED_FILE) || true
+
+.PHONY: helm
+helm: cleanhelm
+ helm template binary-tst-deploy ./ -n flyte --dependency-update --debug --create-namespace -a rbac.authorization.k8s.io/v1 -a networking.k8s.io/v1/Ingress -a apiextensions.k8s.io/v1/CustomResourceDefinition > $(GENERATED_FILE)
+
+.PHONY: devboxhelm
+devboxhelm: cleandevbox
+ helm template flytedevbox ./ -f flytectldemo.yaml -n flyte --dependency-update --debug --create-namespace -a rbac.authorization.k8s.io/v1 -a networking.k8s.io/v1/Ingress -a apiextensions.k8s.io/v1/CustomResourceDefinition > $(DEVBOX_GENERATED_FILE)
diff --git a/charts/flyte-devbox/templates/NOTES.txt b/charts/flyte-devbox/templates/NOTES.txt
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/charts/flyte-devbox/templates/_helpers.tpl b/charts/flyte-devbox/templates/_helpers.tpl
new file mode 100644
index 00000000000..ebc02a2fc3a
--- /dev/null
+++ b/charts/flyte-devbox/templates/_helpers.tpl
@@ -0,0 +1,106 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "flyte-devbox.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "flyte-devbox.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "flyte-devbox.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "flyte-devbox.labels" -}}
+helm.sh/chart: {{ include "flyte-devbox.chart" . }}
+{{ include "flyte-devbox.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "flyte-devbox.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "flyte-devbox.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "flyte-devbox.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "flyte-devbox.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Name of inline ConfigMap containing additional configuration or overrides for Flyte
+*/}}
+{{- define "flyte-devbox.configuration.inlineConfigMap" -}}
+{{- printf "%s-extra-config" .Release.Name -}}
+{{- end }}
+
+{{/*
+Name of inline ConfigMap containing additional cluster resource templates
+*/}}
+{{- define "flyte-devbox.clusterResourceTemplates.inlineConfigMap" -}}
+{{- printf "%s-extra-cluster-resource-templates" .Release.Name -}}
+{{- end }}
+
+{{/*
+Name of PersistentVolume and PersistentVolumeClaim for PostgreSQL database
+*/}}
+{{- define "flyte-devbox.persistence.dbVolumeName" -}}
+{{- printf "%s-db-storage" .Release.Name -}}
+{{- end }}
+
+{{/*
+Name of PersistentVolume and PersistentVolumeClaim for RustFS
+*/}}
+{{- define "flyte-devbox.persistence.rustfsVolumeName" -}}
+{{- printf "%s-rustfs-storage" .Release.Name -}}
+{{- end }}
+
+
+{{/*
+Selector labels for console
+*/}}
+{{- define "flyte-devbox.consoleSelectorLabels" -}}
+{{ include "flyte-devbox.selectorLabels" . }}
+app.kubernetes.io/component: console
+{{- end }}
+
+{{/*
+Name of development-mode Flyte headless service
+*/}}
+{{- define "flyte-devbox.localHeadlessService" -}}
+{{- printf "%s-local" .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- end }}
diff --git a/charts/flyte-devbox/templates/config/cluster-resource-template-inline-configmap.yaml b/charts/flyte-devbox/templates/config/cluster-resource-template-inline-configmap.yaml
new file mode 100644
index 00000000000..e79fab266fe
--- /dev/null
+++ b/charts/flyte-devbox/templates/config/cluster-resource-template-inline-configmap.yaml
@@ -0,0 +1,7 @@
+{{- if ( index .Values "flyte-binary" "enabled" ) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "flyte-devbox.clusterResourceTemplates.inlineConfigMap" . }}
+ namespace: {{ .Release.Namespace | quote }}
+{{- end }}
diff --git a/charts/flyte-devbox/templates/config/configuration-inline-configmap.yaml b/charts/flyte-devbox/templates/config/configuration-inline-configmap.yaml
new file mode 100644
index 00000000000..9f7e8da2f3d
--- /dev/null
+++ b/charts/flyte-devbox/templates/config/configuration-inline-configmap.yaml
@@ -0,0 +1,7 @@
+{{- if ( index .Values "flyte-binary" "enabled" ) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "flyte-devbox.configuration.inlineConfigMap" . }}
+ namespace: {{ .Release.Namespace | quote }}
+{{- end }}
diff --git a/charts/flyte-devbox/templates/console/deployment.yaml b/charts/flyte-devbox/templates/console/deployment.yaml
new file mode 100644
index 00000000000..ba25912037f
--- /dev/null
+++ b/charts/flyte-devbox/templates/console/deployment.yaml
@@ -0,0 +1,38 @@
+{{- if .Values.sandbox.console.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: flyte-console
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-devbox.labels" . | nindent 4 }}
+spec:
+ replicas: 1
+ selector:
+ matchLabels: {{- include "flyte-devbox.consoleSelectorLabels" . | nindent 6 }}
+ template:
+ metadata:
+ labels: {{- include "flyte-devbox.consoleSelectorLabels" . | nindent 8 }}
+ spec:
+ containers:
+ - name: console
+ {{- with .Values.sandbox.console.image }}
+ image: {{ printf "%s:%s" .repository .tag | quote }}
+ imagePullPolicy: {{ .pullPolicy | quote }}
+ {{- end }}
+ ports:
+ - name: http
+ containerPort: 8080
+ protocol: TCP
+ readinessProbe:
+ httpGet:
+ path: /v2
+ port: http
+ initialDelaySeconds: 5
+ periodSeconds: 10
+ livenessProbe:
+ httpGet:
+ path: /v2
+ port: http
+ initialDelaySeconds: 5
+ periodSeconds: 30
+{{- end }}
diff --git a/charts/flyte-devbox/templates/console/service.yaml b/charts/flyte-devbox/templates/console/service.yaml
new file mode 100644
index 00000000000..0c60a94ff8c
--- /dev/null
+++ b/charts/flyte-devbox/templates/console/service.yaml
@@ -0,0 +1,15 @@
+{{- if .Values.sandbox.console.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: flyte-console
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-devbox.labels" . | nindent 4 }}
+spec:
+ selector: {{- include "flyte-devbox.consoleSelectorLabels" . | nindent 4 }}
+ ports:
+ - name: http
+ port: 80
+ targetPort: http
+ protocol: TCP
+{{- end }}
diff --git a/charts/flyte-devbox/templates/local/endpoint.yaml b/charts/flyte-devbox/templates/local/endpoint.yaml
new file mode 100644
index 00000000000..3da3bb64c35
--- /dev/null
+++ b/charts/flyte-devbox/templates/local/endpoint.yaml
@@ -0,0 +1,19 @@
+{{- if and ( not ( index .Values "flyte-binary" "enabled" ) ) .Values.sandbox.dev }}
+apiVersion: v1
+kind: Endpoints
+metadata:
+ name: {{ include "flyte-devbox.localHeadlessService" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ {{- include "flyte-devbox.labels" . | nindent 4 }}
+subsets:
+ - addresses:
+ - ip: '%{HOST_GATEWAY_IP}%'
+ ports:
+ - name: http
+ port: 8090
+ protocol: TCP
+ - name: webhook
+ port: 9443
+ protocol: TCP
+{{- end }}
diff --git a/charts/flyte-devbox/templates/local/service.yaml b/charts/flyte-devbox/templates/local/service.yaml
new file mode 100644
index 00000000000..d66ff7ae673
--- /dev/null
+++ b/charts/flyte-devbox/templates/local/service.yaml
@@ -0,0 +1,18 @@
+{{- if and ( not ( index .Values "flyte-binary" "enabled" ) ) .Values.sandbox.dev }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "flyte-devbox.localHeadlessService" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ {{- include "flyte-devbox.labels" . | nindent 4 }}
+spec:
+ clusterIP: None
+ ports:
+ - name: http
+ port: 8090
+ protocol: TCP
+ - name: webhook
+ port: 9443
+ protocol: TCP
+{{- end }}
diff --git a/charts/flyte-devbox/templates/proxy/ingress.yaml b/charts/flyte-devbox/templates/proxy/ingress.yaml
new file mode 100644
index 00000000000..ff720192944
--- /dev/null
+++ b/charts/flyte-devbox/templates/proxy/ingress.yaml
@@ -0,0 +1,55 @@
+{{- if or ( index .Values "flyte-binary" "enabled" ) .Values.sandbox.dev }}
+{{- $backendService := "" }}
+{{- if index .Values "flyte-binary" "enabled" }}
+{{- $backendService = printf "%s-http" ( index .Values "flyte-binary" "fullnameOverride" | default "flyte-binary" ) }}
+{{- else }}
+{{- $backendService = include "flyte-devbox.localHeadlessService" . }}
+{{- end }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ name: {{ include "flyte-devbox.fullname" . }}-api
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-devbox.labels" . | nindent 4 }}
+spec:
+ rules:
+ - http:
+ paths:
+ {{- range $path := list "/healthz" "/readyz" }}
+ - path: {{ $path }}
+ pathType: Exact
+ backend:
+ service:
+ name: {{ $backendService }}
+ port:
+ number: 8090
+ {{- end }}
+ - path: /flyteidl2.
+ pathType: Prefix
+ backend:
+ service:
+ name: {{ $backendService }}
+ port:
+ number: 8090
+{{- end }}
+---
+{{- if .Values.sandbox.console.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ name: flyte-console
+ namespace: {{ .Release.Namespace | quote }}
+ labels: {{- include "flyte-devbox.labels" . | nindent 4 }}
+spec:
+ rules:
+ - http:
+ paths:
+ - path: /v2
+ pathType: Prefix
+ backend:
+ service:
+ name: flyte-console
+ port:
+ number: 80
+{{- end }}
+---
diff --git a/charts/flyte-devbox/templates/proxy/traefik-config.yaml b/charts/flyte-devbox/templates/proxy/traefik-config.yaml
new file mode 100644
index 00000000000..5657b622f81
--- /dev/null
+++ b/charts/flyte-devbox/templates/proxy/traefik-config.yaml
@@ -0,0 +1,18 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+ name: traefik
+ namespace: kube-system
+ labels: {{- include "flyte-devbox.labels" . | nindent 4 }}
+spec:
+ valuesContent: |
+ service:
+ type: NodePort
+ ports:
+ web:
+ nodePort: 30080
+ transport:
+ respondingTimeouts:
+ readTimeout: 0
+ websecure:
+ expose: false
diff --git a/charts/flyte-devbox/templates/storage/rustfs/deployment.yaml b/charts/flyte-devbox/templates/storage/rustfs/deployment.yaml
new file mode 100644
index 00000000000..058f41c8151
--- /dev/null
+++ b/charts/flyte-devbox/templates/storage/rustfs/deployment.yaml
@@ -0,0 +1,85 @@
+{{- if .Values.rustfs.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: rustfs
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ {{- include "flyte-devbox.labels" . | nindent 4 }}
+spec:
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: rustfs
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ strategy:
+ type: Recreate
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: rustfs
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ spec:
+ initContainers:
+ - name: volume-permissions
+ image: busybox:latest
+ imagePullPolicy: IfNotPresent
+ command:
+ - /bin/sh
+ - -ec
+ - |
+ chown -R 10001:10001 /data
+ mkdir -p /data/flyte-data
+ chown 10001:10001 /data/flyte-data
+ securityContext:
+ runAsUser: 0
+ volumeMounts:
+ - mountPath: /data
+ name: data
+ containers:
+ - name: rustfs
+ image: rustfs/rustfs:sandbox
+ imagePullPolicy: Never
+ env:
+ - name: RUSTFS_ADDRESS
+ value: "0.0.0.0:9000"
+ - name: RUSTFS_VOLUMES
+ value: "/data"
+ - name: RUSTFS_ACCESS_KEY
+ valueFrom:
+ secretKeyRef:
+ name: rustfs
+ key: access-key
+ - name: RUSTFS_SECRET_KEY
+ valueFrom:
+ secretKeyRef:
+ name: rustfs
+ key: secret-key
+ ports:
+ - containerPort: 9000
+ name: rustfs-api
+ protocol: TCP
+ livenessProbe:
+ tcpSocket:
+ port: rustfs-api
+ initialDelaySeconds: 10
+ periodSeconds: 10
+ failureThreshold: 5
+ readinessProbe:
+ tcpSocket:
+ port: rustfs-api
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ failureThreshold: 5
+ securityContext:
+ runAsUser: 10001
+ runAsNonRoot: true
+ volumeMounts:
+ - mountPath: /data
+ name: data
+ securityContext:
+ fsGroup: 10001
+ volumes:
+ - name: data
+ persistentVolumeClaim:
+ claimName: {{ include "flyte-devbox.persistence.rustfsVolumeName" . }}
+{{- end }}
diff --git a/charts/flyte-devbox/templates/storage/rustfs/pv.yaml b/charts/flyte-devbox/templates/storage/rustfs/pv.yaml
new file mode 100644
index 00000000000..3eb50007680
--- /dev/null
+++ b/charts/flyte-devbox/templates/storage/rustfs/pv.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.rustfs.enabled }}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+ name: {{ include "flyte-devbox.persistence.rustfsVolumeName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ {{- include "flyte-devbox.labels" . | nindent 4 }}
+spec:
+ storageClassName: manual
+ accessModes:
+ - ReadWriteOnce
+ capacity:
+ storage: 1Gi
+ hostPath:
+ path: "/var/lib/flyte/storage/rustfs"
+{{- end }}
diff --git a/charts/flyte-devbox/templates/storage/rustfs/pvc.yaml b/charts/flyte-devbox/templates/storage/rustfs/pvc.yaml
new file mode 100644
index 00000000000..500bb732732
--- /dev/null
+++ b/charts/flyte-devbox/templates/storage/rustfs/pvc.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.rustfs.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: {{ include "flyte-devbox.persistence.rustfsVolumeName" . }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ {{- include "flyte-devbox.labels" . | nindent 4 }}
+spec:
+ storageClassName: manual
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ volumeName: {{ include "flyte-devbox.persistence.rustfsVolumeName" . }}
+{{- end }}
diff --git a/charts/flyte-devbox/templates/storage/rustfs/secret.yaml b/charts/flyte-devbox/templates/storage/rustfs/secret.yaml
new file mode 100644
index 00000000000..3766e0236f9
--- /dev/null
+++ b/charts/flyte-devbox/templates/storage/rustfs/secret.yaml
@@ -0,0 +1,13 @@
+{{- if .Values.rustfs.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+ name: rustfs
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ {{- include "flyte-devbox.labels" . | nindent 4 }}
+type: Opaque
+data:
+ access-key: {{ .Values.rustfs.accessKey | b64enc | quote }}
+ secret-key: {{ .Values.rustfs.secretKey | b64enc | quote }}
+{{- end }}
diff --git a/charts/flyte-devbox/templates/storage/rustfs/service.yaml b/charts/flyte-devbox/templates/storage/rustfs/service.yaml
new file mode 100644
index 00000000000..2b4e80798cc
--- /dev/null
+++ b/charts/flyte-devbox/templates/storage/rustfs/service.yaml
@@ -0,0 +1,19 @@
+{{- if .Values.rustfs.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: rustfs
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ {{- include "flyte-devbox.labels" . | nindent 4 }}
+spec:
+ type: NodePort
+ ports:
+ - name: rustfs-api
+ nodePort: 30002
+ port: 9000
+ targetPort: rustfs-api
+ selector:
+ app.kubernetes.io/name: rustfs
+ app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
diff --git a/charts/flyte-devbox/values.yaml b/charts/flyte-devbox/values.yaml
new file mode 100644
index 00000000000..85af9a39562
--- /dev/null
+++ b/charts/flyte-devbox/values.yaml
@@ -0,0 +1,150 @@
+docker-registry:
+ fullnameOverride: docker-registry
+ enabled: true
+ image:
+ tag: sandbox
+ pullPolicy: Never
+ persistence:
+ enabled: false
+ secrets:
+ haSharedSecret: flytesandboxsecret
+ service:
+ type: NodePort
+ nodePort: 30000
+
+flyte-binary:
+ fullnameOverride: flyte-binary
+ enabled: true
+ configuration:
+ database:
+ host: 'postgresql'
+ password: postgres
+ storage:
+ metadataContainer: flyte-data
+ userDataContainer: flyte-data
+ provider: s3
+ providerConfig:
+ s3:
+ disableSSL: true
+ v2Signing: true
+ endpoint: http://rustfs.{{ .Release.Namespace }}:9000
+ authType: accesskey
+ accessKey: rustfs
+ secretKey: rustfsstorage
+ logging:
+ level: 5
+ inline:
+ task_resources:
+ defaults:
+ cpu: 500m
+ ephemeralStorage: 0
+ gpu: 0
+ memory: 1Gi
+ limits:
+ cpu: 0
+ ephemeralStorage: 0
+ gpu: 0
+ memory: 0
+ storage:
+ signedURL:
+ stowConfigOverride:
+ endpoint: http://localhost:30002
+ plugins:
+ k8s:
+ default-env-vars:
+ - FLYTE_AWS_ENDPOINT: http://rustfs.{{ .Release.Namespace }}:9000
+ - FLYTE_AWS_ACCESS_KEY_ID: rustfs
+ - FLYTE_AWS_SECRET_ACCESS_KEY: rustfsstorage
+ - _U_EP_OVERRIDE: 'flyte-binary-http.{{ .Release.Namespace }}:8090'
+ - _U_INSECURE: "true"
+ - _U_USE_ACTIONS: "1"
+ runs:
+ database:
+ postgres:
+ host: 'postgresql.{{ .Release.Namespace }}'
+ port: 5432
+ dbName: runs
+ user: postgres
+ password: postgres
+ inlineConfigMap: '{{ include "flyte-devbox.configuration.inlineConfigMap" . }}'
+ clusterResourceTemplates:
+ inlineConfigMap: '{{ include "flyte-devbox.clusterResourceTemplates.inlineConfigMap" . }}'
+ deployment:
+ image:
+ repository: flyte-binary-v2
+ tag: sandbox
+ pullPolicy: Never
+ startupProbe:
+ httpGet:
+ path: /healthz
+ port: http
+ failureThreshold: 30
+ periodSeconds: 1
+ livenessProbe:
+ httpGet:
+ path: /healthz
+ port: http
+ readinessProbe:
+ httpGet:
+ path: /readyz
+ port: http
+ periodSeconds: 1
+ waitForDB:
+ image:
+ repository: bitnami/postgresql
+ tag: sandbox
+ pullPolicy: Never
+ rbac:
+ # This is strictly NOT RECOMMENDED in production clusters, and is only for use
+ # within local Flyte sandboxes.
+ # When using cluster resource templates to create additional namespaced roles,
+ # Flyte is required to have a superset of those permissions. To simplify
+ # experimenting with new backend plugins that require additional roles be created
+ # with cluster resource templates (e.g. Spark), we add the following:
+ extraRules:
+ - apiGroups:
+ - '*'
+ resources:
+ - '*'
+ verbs:
+ - '*'
+
+rustfs:
+ enabled: true
+ accessKey: rustfs
+ secretKey: rustfsstorage
+
+postgresql:
+ fullnameOverride: postgresql
+ enabled: true
+ image:
+ tag: sandbox
+ pullPolicy: Never
+ auth:
+ postgresPassword: postgres
+ shmVolume:
+ enabled: false
+ primary:
+ service:
+ type: NodePort
+ nodePorts:
+ postgresql: 30001
+ persistence:
+ enabled: true
+ existingClaim: '{{ include "flyte-devbox.persistence.dbVolumeName" . }}'
+ volumePermissions:
+ enabled: true
+ image:
+ tag: sandbox
+ pullPolicy: Never
+
+sandbox:
+ # dev Routes requests to an instance of Flyte running locally on a developer's
+ # development environment. This is only usable if the flyte-binary chart is disabled.
+ dev: false
+ console:
+ enabled: true
+ image:
+ repository: docker.io/unionai-oss/flyteconsole-v2
+ tag: sandbox
+ pullPolicy: Never
diff --git a/config/k3d/cluster.yaml b/config/k3d/cluster.yaml
new file mode 100644
index 00000000000..68a19f4acb7
--- /dev/null
+++ b/config/k3d/cluster.yaml
@@ -0,0 +1,23 @@
+apiVersion: k3d.io/v1alpha5
+kind: Simple
+metadata:
+ name: ${CLUSTER_NAME}
+servers: 1
+agents: 0
+options:
+ k3d:
+ disableLoadbalancer: true
+ k3s:
+ extraArgs:
+ - arg: --disable=servicelb
+ nodeFilters:
+ - server:*
+ kubeconfig:
+ updateDefaultKubeconfig: true
+ switchCurrentContext: true
+# Inject host gateway IP so pods can reach services running on the host
+# (e.g. manager at :8090, minio at :9000)
+hostAliases:
+ - ip: ${HOST_GATEWAY_IP}
+ hostnames:
+ - flyte-host
diff --git a/dataproxy/README.md b/dataproxy/README.md
new file mode 100644
index 00000000000..5ff0ca860f2
--- /dev/null
+++ b/dataproxy/README.md
@@ -0,0 +1,137 @@
+# Data Proxy
+
+Data proxy is the service that communicates with the data storage to handle operations like:
+- Generate signed URLs for uploading data
+- Generate download URLs for retrieving data
+- Manage data access and permissions
+
+Data proxy lives in the Flyte control plane, while the data storage lives in the data plane. Data proxy is the bridge
+to request URLs for uploading/downloading data to any data storage you have.
+
+## Quick Start
+
+For production deployment:
+
+1. Configure your storage backend (S3, RustFS, GCS, etc.)
+2. Create a configuration file based on `config/config.example.yaml`
+3. Run the service:
+
+```bash
+go run cmd/main.go --config config/config.yaml
+```
+
+The service will start on port 8088 by default. You can override with `--port` and `--host` flags.
+
+## Configuration
+
+The data proxy service uses a YAML configuration file. See [config.example.yaml](config/config.example.yaml) for a complete example.
+
+### Data Proxy Settings
+
+| Field | Description | Default |
+|-------|-------------|---------|
+| `dataproxy.upload.maxSize` | Maximum allowed upload size | `100Mi` |
+| `dataproxy.upload.maxExpiresIn` | Maximum expiration time for signed upload URLs | `1h` |
+| `dataproxy.upload.defaultFileNameLength` | Default length for auto-generated filenames | `20` |
+| `dataproxy.upload.storagePrefix` | Prefix for all uploaded files in storage | `uploads` |
+| `dataproxy.download.maxExpiresIn` | Maximum expiration time for download URLs | `1h` |
+
+### Storage Backend Settings
+
+| Field | Description | Options |
+|-------|-------------|---------|
+| `storage.type` | Storage backend type | `s3`, `stow`, `local`, `mem` |
+| `storage.container` | Initial bucket/container name | - |
+| `storage.enable-multicontainer` | Allow access to multiple buckets | `true`/`false` |
+| `storage.connection.endpoint` | Storage endpoint URL | - |
+| `storage.connection.auth-type` | Authentication method | `iam`, `accesskey` |
+| `storage.connection.access-key` | Access key (when using `accesskey` auth) | - |
+| `storage.connection.secret-key` | Secret key (when using `accesskey` auth) | - |
+| `storage.connection.region` | Storage region | `us-east-1` |
+| `storage.connection.disable-ssl` | Disable SSL (for local dev only) | `true`/`false` |
+
+### RustFS Configuration
+
+For RustFS (used in local development and sandbox):
+
+```yaml
+storage:
+ type: stow
+ container: "flyte-data"
+ stow:
+ kind: s3
+ config:
+ auth_type: accesskey
+ access_key_id: "rustfs"
+ secret_key: "rustfsstorage"
+ endpoint: "http://rustfs.flyte-dataproxy.svc.cluster.local:9000"
+ region: "us-east-1"
+ disable_ssl: "true"
+```
+
+### S3-Compatible Storage
+
+For AWS S3 or S3-compatible storage:
+
+```yaml
+storage:
+ type: stow
+ container: "my-flyte-bucket"
+ stow:
+ kind: s3
+ config:
+ auth_type: iam
+ region: "us-west-2"
+```
+
+## Troubleshooting
+
+### RustFS Connection Issues
+
+1. **Check RustFS is running:**
+ ```bash
+ kubectl get pods -n
+ ```
+
+2. **Check RustFS logs:**
+ ```bash
+ kubectl logs -n -l app=rustfs
+ ```
+
+3. **Verify network connectivity:**
+ ```bash
+ kubectl run -it --rm debug --image=curlimages/curl --restart=Never -- \
+ curl http://rustfs.flyte-dataproxy.svc.cluster.local:9000
+ ```
+
+### Data Proxy Service Issues
+
+1. **Check configuration:**
+ - Verify the storage endpoint is correct
+ - Ensure credentials are valid
+ - Check that the bucket/container exists
+
+2. **Enable debug logging:**
+ ```yaml
+ logger:
+ level: 5 # Debug level
+ show-source: true
+ ```
+
+3. **Common errors:**
+ - `connection refused`: RustFS service not reachable
+ - `access denied`: Invalid credentials
+ - `bucket not found`: Container doesn't exist (enable auto-creation or create manually)
+
+## Production Deployment
+
+For production use:
+
+1. **Use a production-grade storage backend** (AWS S3, GCS, Azure Blob Storage, or RustFS in distributed mode)
+2. **Enable TLS/SSL** for all connections
+3. **Use IAM roles** instead of static credentials when possible
+4. **Configure resource limits** and proper scaling
+5. **Set up monitoring and alerting** for storage operations
+6. **Review security settings** for signed URL expiration times
+
+See the Flyte documentation for production deployment guides.
diff --git a/dataproxy/cmd/main.go b/dataproxy/cmd/main.go
new file mode 100644
index 00000000000..e224d50efde
--- /dev/null
+++ b/dataproxy/cmd/main.go
@@ -0,0 +1,37 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/app"
+ "github.com/flyteorg/flyte/v2/dataproxy"
+ "github.com/flyteorg/flyte/v2/flytestdlib/contextutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils/labeled"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+)
+
+func main() {
+ a := &app.App{
+ Name: "dataproxy-service",
+ Short: "Data Proxy Service for Flyte",
+ Setup: func(ctx context.Context, sc *app.SetupContext) error {
+ sc.Host = "0.0.0.0"
+ sc.Port = 8088
+
+ labeled.SetMetricKeys(contextutils.ProjectKey, contextutils.DomainKey, contextutils.WorkflowIDKey, contextutils.TaskIDKey)
+ dataStore, err := storage.NewDataStore(storage.GetConfig(), promutils.NewTestScope())
+ if err != nil {
+ return fmt.Errorf("failed to initialize storage: %w", err)
+ }
+ sc.DataStore = dataStore
+
+ return dataproxy.Setup(ctx, sc)
+ },
+ }
+ if err := a.Run(); err != nil {
+ os.Exit(1)
+ }
+}
diff --git a/dataproxy/config/config.example.yaml b/dataproxy/config/config.example.yaml
new file mode 100644
index 00000000000..789105399e7
--- /dev/null
+++ b/dataproxy/config/config.example.yaml
@@ -0,0 +1,43 @@
+# Example configuration for Data Proxy with RustFS storage backend
+# This configuration is for local development and testing purposes.
+
+# Data Proxy Configuration
+dataproxy:
+ upload:
+ # Maximum size for uploads (100MB)
+ maxSize: "100Mi"
+ # Maximum expiration time for signed URLs (1 hour)
+ maxExpiresIn: 1h
+ # Default filename length when auto-generating names
+ defaultFileNameLength: 20
+ # Storage prefix for all uploads
+ storagePrefix: "uploads"
+
+ download:
+ # Maximum expiration time for download URLs (1 hour)
+ maxExpiresIn: 1h
+
+# Storage Configuration
+storage:
+ # Type of storage backend (options: s3, minio, local, mem, stow)
+ type: minio
+
+ # Initial bucket/container name
+ # This bucket will be created automatically if it doesn't exist
+ container: "flyte-data"
+
+ # Enable multi-container support
+ # When true, allows accessing multiple buckets
+ enable-multicontainer: false
+
+ # Stow Configuration (recommended)
+ # This is the modern way to configure storage backends
+ stow:
+ kind: "s3"
+ config:
+ auth_type: "accesskey"
+ access_key_id: "rustfs"
+ secret_key: "rustfsstorage"
+ region: "us-east-1"
+ endpoint: "http://localhost:9000"
+ disable_ssl: "true"
diff --git a/dataproxy/config/config.go b/dataproxy/config/config.go
new file mode 100644
index 00000000000..33e59794fe2
--- /dev/null
+++ b/dataproxy/config/config.go
@@ -0,0 +1,45 @@
+package config
+
+import (
+ "github.com/flyteorg/flyte/v2/flytestdlib/config"
+ "k8s.io/apimachinery/pkg/api/resource"
+)
+
+const configSectionKey = "dataproxy"
+
+//go:generate pflags DataProxyConfig --default-var=defaultConfig
+
+var defaultConfig = &DataProxyConfig{
+ Upload: DataProxyUploadConfig{
+ MaxSize: resource.MustParse("100Mi"),
+ MaxExpiresIn: config.Duration{Duration: 3600000000000}, // 1 hour
+ DefaultFileNameLength: 20,
+ StoragePrefix: "uploads",
+ },
+ Download: DataProxyDownloadConfig{
+ MaxExpiresIn: config.Duration{Duration: 3600000000000}, // 1 hour
+ },
+}
+
+var configSection = config.MustRegisterSection(configSectionKey, defaultConfig)
+
+type DataProxyConfig struct {
+ Upload DataProxyUploadConfig `json:"upload" pflag:",Defines data proxy upload configuration."`
+ Download DataProxyDownloadConfig `json:"download" pflag:",Defines data proxy download configuration."`
+}
+
+// GetConfig returns the parsed data proxy configuration
+func GetConfig() *DataProxyConfig {
+ return configSection.GetConfig().(*DataProxyConfig)
+}
+
+type DataProxyDownloadConfig struct {
+ MaxExpiresIn config.Duration `json:"maxExpiresIn" pflag:",Maximum allowed expiration duration."`
+}
+
+type DataProxyUploadConfig struct {
+ MaxSize resource.Quantity `json:"maxSize" pflag:",Maximum allowed upload size."`
+ MaxExpiresIn config.Duration `json:"maxExpiresIn" pflag:",Maximum allowed expiration duration."`
+ DefaultFileNameLength int `json:"defaultFileNameLength" pflag:",Default length for the generated file name if file name not provided in the request."`
+ StoragePrefix string `json:"storagePrefix" pflag:",Storage prefix to use for all upload requests."`
+}
diff --git a/dataproxy/config/dataproxyconfig_flags.go b/dataproxy/config/dataproxyconfig_flags.go
new file mode 100755
index 00000000000..54c0c0339c9
--- /dev/null
+++ b/dataproxy/config/dataproxyconfig_flags.go
@@ -0,0 +1,59 @@
+// Code generated by go generate; DO NOT EDIT.
+// This file was generated by robots.
+
+package config
+
+import (
+ "encoding/json"
+ "reflect"
+
+ "fmt"
+
+ "github.com/spf13/pflag"
+)
+
+// If v is a pointer, it will get its element value or the zero value of the element type.
+// If v is not a pointer, it will return it as is.
+func (DataProxyConfig) elemValueOrNil(v interface{}) interface{} {
+ if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr {
+ if reflect.ValueOf(v).IsNil() {
+ return reflect.Zero(t.Elem()).Interface()
+ } else {
+ return reflect.ValueOf(v).Interface()
+ }
+ } else if v == nil {
+ return reflect.Zero(t).Interface()
+ }
+
+ return v
+}
+
+func (DataProxyConfig) mustJsonMarshal(v interface{}) string {
+ raw, err := json.Marshal(v)
+ if err != nil {
+ panic(err)
+ }
+
+ return string(raw)
+}
+
+func (DataProxyConfig) mustMarshalJSON(v json.Marshaler) string {
+ raw, err := v.MarshalJSON()
+ if err != nil {
+ panic(err)
+ }
+
+ return string(raw)
+}
+
+// GetPFlagSet will return strongly types pflags for all fields in DataProxyConfig and its nested types. The format of the
+// flags is json-name.json-sub-name... etc.
+func (cfg DataProxyConfig) GetPFlagSet(prefix string) *pflag.FlagSet {
+ cmdFlags := pflag.NewFlagSet("DataProxyConfig", pflag.ExitOnError)
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "upload.maxSize"), defaultConfig.Upload.MaxSize.String(), "Maximum allowed upload size.")
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "upload.maxExpiresIn"), defaultConfig.Upload.MaxExpiresIn.String(), "Maximum allowed expiration duration.")
+ cmdFlags.Int(fmt.Sprintf("%v%v", prefix, "upload.defaultFileNameLength"), defaultConfig.Upload.DefaultFileNameLength, "Default length for the generated file name if file name not provided in the request.")
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "upload.storagePrefix"), defaultConfig.Upload.StoragePrefix, "Storage prefix to use for all upload requests.")
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "download.maxExpiresIn"), defaultConfig.Download.MaxExpiresIn.String(), "Maximum allowed expiration duration.")
+ return cmdFlags
+}
diff --git a/dataproxy/config/dataproxyconfig_flags_test.go b/dataproxy/config/dataproxyconfig_flags_test.go
new file mode 100755
index 00000000000..da2e1267924
--- /dev/null
+++ b/dataproxy/config/dataproxyconfig_flags_test.go
@@ -0,0 +1,172 @@
+// Code generated by go generate; DO NOT EDIT.
+// This file was generated by robots.
+
+package config
+
+import (
+ "encoding/json"
+ "fmt"
+ "reflect"
+ "strings"
+ "testing"
+
+ "github.com/mitchellh/mapstructure"
+ "github.com/stretchr/testify/assert"
+)
+
+var dereferencableKindsDataProxyConfig = map[reflect.Kind]struct{}{
+ reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {},
+}
+
+// Checks if t is a kind that can be dereferenced to get its underlying type.
+func canGetElementDataProxyConfig(t reflect.Kind) bool {
+ _, exists := dereferencableKindsDataProxyConfig[t]
+ return exists
+}
+
+// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the
+// object. Otherwise, it'll just pass on the original data.
+func jsonUnmarshalerHookDataProxyConfig(_, to reflect.Type, data interface{}) (interface{}, error) {
+ unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem()
+ if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) ||
+ (canGetElementDataProxyConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) {
+
+ raw, err := json.Marshal(data)
+ if err != nil {
+ fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err)
+ return data, nil
+ }
+
+ res := reflect.New(to).Interface()
+ err = json.Unmarshal(raw, &res)
+ if err != nil {
+ fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err)
+ return data, nil
+ }
+
+ return res, nil
+ }
+
+ return data, nil
+}
+
+func decode_DataProxyConfig(input, result interface{}) error {
+ config := &mapstructure.DecoderConfig{
+ TagName: "json",
+ WeaklyTypedInput: true,
+ Result: result,
+ DecodeHook: mapstructure.ComposeDecodeHookFunc(
+ mapstructure.StringToTimeDurationHookFunc(),
+ mapstructure.StringToSliceHookFunc(","),
+ jsonUnmarshalerHookDataProxyConfig,
+ ),
+ }
+
+ decoder, err := mapstructure.NewDecoder(config)
+ if err != nil {
+ return err
+ }
+
+ return decoder.Decode(input)
+}
+
+func join_DataProxyConfig(arr interface{}, sep string) string {
+ listValue := reflect.ValueOf(arr)
+ strs := make([]string, 0, listValue.Len())
+ for i := 0; i < listValue.Len(); i++ {
+ strs = append(strs, fmt.Sprintf("%v", listValue.Index(i)))
+ }
+
+ return strings.Join(strs, sep)
+}
+
+func testDecodeJson_DataProxyConfig(t *testing.T, val, result interface{}) {
+ assert.NoError(t, decode_DataProxyConfig(val, result))
+}
+
+func testDecodeRaw_DataProxyConfig(t *testing.T, vStringSlice, result interface{}) {
+ assert.NoError(t, decode_DataProxyConfig(vStringSlice, result))
+}
+
+func TestDataProxyConfig_GetPFlagSet(t *testing.T) {
+ val := DataProxyConfig{}
+ cmdFlags := val.GetPFlagSet("")
+ assert.True(t, cmdFlags.HasFlags())
+}
+
+func TestDataProxyConfig_SetFlags(t *testing.T) {
+ actual := DataProxyConfig{}
+ cmdFlags := actual.GetPFlagSet("")
+ assert.True(t, cmdFlags.HasFlags())
+
+ t.Run("Test_upload.maxSize", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := defaultConfig.Upload.MaxSize.String()
+
+ cmdFlags.Set("upload.maxSize", testValue)
+ if vString, err := cmdFlags.GetString("upload.maxSize"); err == nil {
+ testDecodeJson_DataProxyConfig(t, fmt.Sprintf("%v", vString), &actual.Upload.MaxSize)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_upload.maxExpiresIn", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := defaultConfig.Upload.MaxExpiresIn.String()
+
+ cmdFlags.Set("upload.maxExpiresIn", testValue)
+ if vString, err := cmdFlags.GetString("upload.maxExpiresIn"); err == nil {
+ testDecodeJson_DataProxyConfig(t, fmt.Sprintf("%v", vString), &actual.Upload.MaxExpiresIn)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_upload.defaultFileNameLength", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("upload.defaultFileNameLength", testValue)
+ if vInt, err := cmdFlags.GetInt("upload.defaultFileNameLength"); err == nil {
+ testDecodeJson_DataProxyConfig(t, fmt.Sprintf("%v", vInt), &actual.Upload.DefaultFileNameLength)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_upload.storagePrefix", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("upload.storagePrefix", testValue)
+ if vString, err := cmdFlags.GetString("upload.storagePrefix"); err == nil {
+ testDecodeJson_DataProxyConfig(t, fmt.Sprintf("%v", vString), &actual.Upload.StoragePrefix)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_download.maxExpiresIn", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := defaultConfig.Download.MaxExpiresIn.String()
+
+ cmdFlags.Set("download.maxExpiresIn", testValue)
+ if vString, err := cmdFlags.GetString("download.maxExpiresIn"); err == nil {
+ testDecodeJson_DataProxyConfig(t, fmt.Sprintf("%v", vString), &actual.Download.MaxExpiresIn)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+}
diff --git a/dataproxy/logs/k8s_log_streamer.go b/dataproxy/logs/k8s_log_streamer.go
new file mode 100644
index 00000000000..15409050370
--- /dev/null
+++ b/dataproxy/logs/k8s_log_streamer.go
@@ -0,0 +1,178 @@
+package logs
+
+import (
+ "bufio"
+ "context"
+ "fmt"
+ "io"
+ "strings"
+ "time"
+
+ "connectrpc.com/connect"
+ corev1 "k8s.io/api/core/v1"
+ k8serrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/rest"
+
+ "google.golang.org/protobuf/types/known/timestamppb"
+
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/dataproxy"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/logs/dataplane"
+)
+
+const (
+ logBatchSize = 100
+ defaultInitialLines = int64(1000)
+)
+
+// K8sLogStreamer streams logs directly from Kubernetes pods.
+type K8sLogStreamer struct {
+ clientset kubernetes.Interface
+}
+
+// NewK8sLogStreamer creates a K8sLogStreamer from a Kubernetes REST config.
+// It clears the timeout so that long-lived log streams are not interrupted.
+func NewK8sLogStreamer(k8sConfig *rest.Config) (*K8sLogStreamer, error) {
+ cfg := rest.CopyConfig(k8sConfig)
+ cfg.Timeout = 0
+ clientset, err := kubernetes.NewForConfig(cfg)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create kubernetes clientset: %w", err)
+ }
+ return &K8sLogStreamer{clientset: clientset}, nil
+}
+
+// TailLogs streams log lines for the given LogContext from a Kubernetes pod.
+func (s *K8sLogStreamer) TailLogs(ctx context.Context, logContext *core.LogContext, stream *connect.ServerStream[dataproxy.TailLogsResponse]) error {
+ pod, container, err := GetPrimaryPodAndContainer(logContext)
+ if err != nil {
+ return connect.NewError(connect.CodeNotFound, err)
+ }
+
+ tailLines := defaultInitialLines
+ opts := &corev1.PodLogOptions{
+ Container: container.GetContainerName(),
+ Follow: true,
+ Timestamps: true,
+ TailLines: &tailLines,
+ }
+
+ // Set SinceTime from container start time if available.
+ // When SinceTime is set, it takes precedence and we clear TailLines
+ // to stream all logs from that point forward.
+ if startTime := container.GetProcess().GetContainerStartTime(); startTime != nil {
+ t := metav1.NewTime(startTime.AsTime())
+ opts.SinceTime = &t
+ opts.TailLines = nil
+ }
+
+ // Only follow logs when the pod is actively running. For pending or
+ // terminated pods, disable follow so existing logs are returned immediately.
+ podObj, err := s.clientset.CoreV1().Pods(pod.GetNamespace()).Get(ctx, pod.GetPodName(), metav1.GetOptions{})
+ if err != nil {
+ if k8serrors.IsNotFound(err) {
+ return connect.NewError(connect.CodeNotFound, fmt.Errorf("pod %s not found in namespace %s", pod.GetPodName(), pod.GetNamespace()))
+ }
+ return connect.NewError(connect.CodeInternal, fmt.Errorf("failed to get pod: %w", err))
+ }
+ opts.Follow = podObj.Status.Phase == corev1.PodRunning
+
+ // Create a context without the incoming gRPC deadline so long-lived follow
+ // streams are not killed by a short client/proxy timeout. Cancellation is
+ // still propagated so the stream closes when the client disconnects.
+ streamCtx, streamCancel := context.WithCancel(context.Background())
+ defer streamCancel()
+ stop := context.AfterFunc(ctx, streamCancel)
+ defer stop()
+
+ logStream, err := s.clientset.CoreV1().Pods(pod.GetNamespace()).GetLogs(pod.GetPodName(), opts).Stream(streamCtx)
+ if err != nil {
+ return connect.NewError(connect.CodeInternal, fmt.Errorf("failed to stream pod logs: %w", err))
+ }
+ defer logStream.Close()
+
+ reader := bufio.NewReader(logStream)
+
+ lines := make([]*dataplane.LogLine, 0, logBatchSize)
+ var readErr error
+
+ for {
+ line, err := reader.ReadString('\n')
+ if len(line) > 0 {
+ // Trim trailing newline(s) including possible CRLF.
+ line = strings.TrimRight(line, "\r\n")
+ logLine := parseLogLine(line)
+ lines = append(lines, logLine)
+
+ if len(lines) >= logBatchSize {
+ if sendErr := stream.Send(&dataproxy.TailLogsResponse{
+ Logs: []*dataproxy.TailLogsResponse_Logs{
+ {Lines: lines},
+ },
+ }); sendErr != nil {
+ return sendErr
+ }
+ lines = make([]*dataplane.LogLine, 0, logBatchSize)
+ }
+ }
+ if err != nil {
+ if err != io.EOF {
+ readErr = err
+ }
+ break
+ }
+
+ // Flush buffered lines when no more data is immediately available.
+ // Without this, lines sit in the buffer while ReadString blocks
+ // waiting for the next newline (e.g. pod is sleeping).
+ if len(lines) > 0 && reader.Buffered() == 0 {
+ if sendErr := stream.Send(&dataproxy.TailLogsResponse{
+ Logs: []*dataproxy.TailLogsResponse_Logs{
+ {Lines: lines},
+ },
+ }); sendErr != nil {
+ return sendErr
+ }
+ lines = make([]*dataplane.LogLine, 0, logBatchSize)
+ }
+ }
+
+ // Send remaining lines.
+ if len(lines) > 0 {
+ if err := stream.Send(&dataproxy.TailLogsResponse{
+ Logs: []*dataproxy.TailLogsResponse_Logs{
+ {Lines: lines},
+ },
+ }); err != nil {
+ return err
+ }
+ }
+
+ // Return error for non-EOF read failures (unless context was canceled).
+ if readErr != nil && ctx.Err() == nil {
+ return connect.NewError(connect.CodeInternal, fmt.Errorf("error reading log stream: %w", readErr))
+ }
+
+ return nil
+}
+
+// parseLogLine splits a K8s log line into timestamp and message.
+// K8s log lines with timestamps are formatted as: "2006-01-02T15:04:05.999999999Z message"
+func parseLogLine(line string) *dataplane.LogLine {
+ if idx := strings.IndexByte(line, ' '); idx > 0 {
+ if t, err := time.Parse(time.RFC3339Nano, line[:idx]); err == nil {
+ return &dataplane.LogLine{
+ Originator: dataplane.LogLineOriginator_USER,
+ Timestamp: timestamppb.New(t),
+ Message: line[idx+1:],
+ }
+ }
+ }
+
+ return &dataplane.LogLine{
+ Originator: dataplane.LogLineOriginator_USER,
+ Message: line,
+ }
+}
diff --git a/dataproxy/logs/k8s_log_streamer_test.go b/dataproxy/logs/k8s_log_streamer_test.go
new file mode 100644
index 00000000000..faa0271319c
--- /dev/null
+++ b/dataproxy/logs/k8s_log_streamer_test.go
@@ -0,0 +1,182 @@
+package logs
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "connectrpc.com/connect"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes/fake"
+
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/logs/dataplane"
+)
+
+func TestParseLogLine_WithTimestamp(t *testing.T) {
+ line := "2024-01-15T10:30:00.123456789Z Hello, world!"
+ logLine := parseLogLine(line)
+
+ assert.Equal(t, "Hello, world!", logLine.Message)
+ assert.NotNil(t, logLine.Timestamp)
+ expected := time.Date(2024, 1, 15, 10, 30, 0, 123456789, time.UTC)
+ assert.Equal(t, expected, logLine.Timestamp.AsTime())
+ assert.Equal(t, dataplane.LogLineOriginator_USER, logLine.Originator)
+}
+
+func TestParseLogLine_WithoutTimestamp(t *testing.T) {
+ line := "just a plain log message"
+ logLine := parseLogLine(line)
+
+ assert.Equal(t, "just a plain log message", logLine.Message)
+ assert.Nil(t, logLine.Timestamp)
+ assert.Equal(t, dataplane.LogLineOriginator_USER, logLine.Originator)
+}
+
+func TestParseLogLine_MalformedTimestamp(t *testing.T) {
+ line := "not-a-timestamp some message"
+ logLine := parseLogLine(line)
+
+ assert.Equal(t, "not-a-timestamp some message", logLine.Message)
+ assert.Nil(t, logLine.Timestamp)
+}
+
+func TestParseLogLine_EmptyMessage(t *testing.T) {
+ line := "2024-01-15T10:30:00Z "
+ logLine := parseLogLine(line)
+
+ assert.Equal(t, "", logLine.Message)
+ assert.NotNil(t, logLine.Timestamp)
+}
+
+func TestGetPrimaryPodAndContainer_HappyPath(t *testing.T) {
+ logCtx := &core.LogContext{
+ PrimaryPodName: "my-pod",
+ Pods: []*core.PodLogContext{
+ {
+ PodName: "my-pod",
+ Namespace: "default",
+ PrimaryContainerName: "main",
+ Containers: []*core.ContainerContext{
+ {ContainerName: "main"},
+ {ContainerName: "sidecar"},
+ },
+ },
+ },
+ }
+
+ pod, container, err := GetPrimaryPodAndContainer(logCtx)
+ assert.NoError(t, err)
+ assert.Equal(t, "my-pod", pod.GetPodName())
+ assert.Equal(t, "default", pod.GetNamespace())
+ assert.Equal(t, "main", container.GetContainerName())
+}
+
+func TestGetPrimaryPodAndContainer_EmptyPodName(t *testing.T) {
+ logCtx := &core.LogContext{
+ PrimaryPodName: "",
+ }
+
+ _, _, err := GetPrimaryPodAndContainer(logCtx)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "primary pod name is empty")
+}
+
+func TestGetPrimaryPodAndContainer_PodNotFound(t *testing.T) {
+ logCtx := &core.LogContext{
+ PrimaryPodName: "missing-pod",
+ Pods: []*core.PodLogContext{
+ {PodName: "other-pod"},
+ },
+ }
+
+ _, _, err := GetPrimaryPodAndContainer(logCtx)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "not found in log context")
+}
+
+func TestGetPrimaryPodAndContainer_ContainerNotFound(t *testing.T) {
+ logCtx := &core.LogContext{
+ PrimaryPodName: "my-pod",
+ Pods: []*core.PodLogContext{
+ {
+ PodName: "my-pod",
+ PrimaryContainerName: "missing-container",
+ Containers: []*core.ContainerContext{
+ {ContainerName: "other"},
+ },
+ },
+ },
+ }
+
+ _, _, err := GetPrimaryPodAndContainer(logCtx)
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "primary container")
+}
+
+func newTestLogContext(podName, namespace, containerName string) *core.LogContext {
+ return &core.LogContext{
+ PrimaryPodName: podName,
+ Pods: []*core.PodLogContext{
+ {
+ PodName: podName,
+ Namespace: namespace,
+ PrimaryContainerName: containerName,
+ Containers: []*core.ContainerContext{
+ {ContainerName: containerName},
+ },
+ },
+ },
+ }
+}
+
+func TestTailLogs_PodNotFound(t *testing.T) {
+ clientset := fake.NewSimpleClientset() // no pods
+
+ streamer := &K8sLogStreamer{clientset: clientset}
+ logCtx := newTestLogContext("missing-pod", "default", "main")
+
+ err := streamer.TailLogs(context.Background(), logCtx, nil)
+ require.Error(t, err)
+ assert.Equal(t, connect.CodeNotFound, connect.CodeOf(err))
+ assert.Contains(t, err.Error(), "not found")
+}
+
+func TestTailLogs_FollowSetBasedOnPodPhase(t *testing.T) {
+ tests := []struct {
+ name string
+ phase corev1.PodPhase
+ wantFollow bool
+ }{
+ {"running pod should follow", corev1.PodRunning, true},
+ {"succeeded pod should not follow", corev1.PodSucceeded, false},
+ {"failed pod should not follow", corev1.PodFailed, false},
+ {"pending pod should not follow", corev1.PodPending, false},
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ clientset := fake.NewSimpleClientset(&corev1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "my-pod",
+ Namespace: "default",
+ },
+ Status: corev1.PodStatus{
+ Phase: tt.phase,
+ },
+ })
+
+ // Verify we can fetch the pod and the phase is correct.
+ podObj, err := clientset.CoreV1().Pods("default").Get(context.Background(), "my-pod", metav1.GetOptions{})
+ require.NoError(t, err)
+ assert.Equal(t, tt.phase, podObj.Status.Phase)
+
+ // Verify the follow logic: Follow should only be true when phase is Running.
+ gotFollow := podObj.Status.Phase == corev1.PodRunning
+ assert.Equal(t, tt.wantFollow, gotFollow)
+ })
+ }
+}
diff --git a/dataproxy/logs/log_streamer.go b/dataproxy/logs/log_streamer.go
new file mode 100644
index 00000000000..8732754fc75
--- /dev/null
+++ b/dataproxy/logs/log_streamer.go
@@ -0,0 +1,40 @@
+package logs
+
+import (
+ "context"
+ "fmt"
+
+ "connectrpc.com/connect"
+ "github.com/samber/lo"
+
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/dataproxy"
+)
+
+// LogStreamer abstracts log fetching from different backends.
+type LogStreamer interface {
+ TailLogs(ctx context.Context, logContext *core.LogContext, stream *connect.ServerStream[dataproxy.TailLogsResponse]) error
+}
+
+// GetPrimaryPodAndContainer finds the primary pod and container from a LogContext.
+func GetPrimaryPodAndContainer(logContext *core.LogContext) (*core.PodLogContext, *core.ContainerContext, error) {
+ if logContext.GetPrimaryPodName() == "" {
+ return nil, nil, fmt.Errorf("primary pod name is empty in log context")
+ }
+
+ pod, found := lo.Find(logContext.GetPods(), func(pod *core.PodLogContext) bool {
+ return pod.GetPodName() == logContext.GetPrimaryPodName()
+ })
+ if !found {
+ return nil, nil, fmt.Errorf("primary pod %s not found in log context", logContext.GetPrimaryPodName())
+ }
+
+ container, found := lo.Find(pod.GetContainers(), func(c *core.ContainerContext) bool {
+ return c.GetContainerName() == pod.GetPrimaryContainerName()
+ })
+ if !found {
+ return nil, nil, fmt.Errorf("primary container %s not found in pod %s", pod.GetPrimaryContainerName(), pod.GetPodName())
+ }
+
+ return pod, container, nil
+}
diff --git a/dataproxy/service/cluster_service.go b/dataproxy/service/cluster_service.go
new file mode 100644
index 00000000000..0c1a684c0d3
--- /dev/null
+++ b/dataproxy/service/cluster_service.go
@@ -0,0 +1,32 @@
+package service
+
+import (
+ "context"
+
+ "connectrpc.com/connect"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/cluster"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/cluster/clusterconnect"
+)
+
+type ClusterService struct {
+ clusterconnect.UnimplementedClusterServiceHandler
+}
+
+func NewClusterService() *ClusterService {
+ return &ClusterService{}
+}
+
+var _ clusterconnect.ClusterServiceHandler = (*ClusterService)(nil)
+
+func (s *ClusterService) SelectCluster(
+ ctx context.Context,
+ req *connect.Request[cluster.SelectClusterRequest],
+) (*connect.Response[cluster.SelectClusterResponse], error) {
+ requestHost := req.Header().Get("Host")
+ logger.Debugf(ctx, "Request Host: %s", requestHost)
+ return connect.NewResponse(&cluster.SelectClusterResponse{
+ ClusterEndpoint: requestHost,
+ }), nil
+}
diff --git a/dataproxy/service/dataproxy_service.go b/dataproxy/service/dataproxy_service.go
new file mode 100644
index 00000000000..edfd49ae230
--- /dev/null
+++ b/dataproxy/service/dataproxy_service.go
@@ -0,0 +1,532 @@
+package service
+
+import (
+ "context"
+ "encoding/base32"
+ "encoding/base64"
+ "fmt"
+ "hash/fnv"
+ "slices"
+ "strings"
+ "time"
+
+ "connectrpc.com/connect"
+ "github.com/flyteorg/stow"
+ "github.com/samber/lo"
+ "golang.org/x/sync/errgroup"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/durationpb"
+ "google.golang.org/protobuf/types/known/timestamppb"
+
+ "github.com/flyteorg/flyte/v2/dataproxy/config"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common"
+ flyteIdlCore "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+ "github.com/flyteorg/flyte/v2/dataproxy/logs"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/dataproxy"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/dataproxy/dataproxyconnect"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/task"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/task/taskconnect"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/trigger"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/trigger/triggerconnect"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow/workflowconnect"
+)
+
+type Service struct {
+ dataproxyconnect.UnimplementedDataProxyServiceHandler
+
+ cfg config.DataProxyConfig
+ dataStore *storage.DataStore
+ taskClient taskconnect.TaskServiceClient
+ triggerClient triggerconnect.TriggerServiceClient
+ runClient workflowconnect.RunServiceClient
+ logStreamer logs.LogStreamer
+}
+
+// NewService creates a new DataProxyService instance.
+func NewService(cfg config.DataProxyConfig, dataStore *storage.DataStore, taskClient taskconnect.TaskServiceClient, triggerClient triggerconnect.TriggerServiceClient, runClient workflowconnect.RunServiceClient, logStreamer logs.LogStreamer) *Service {
+ return &Service{
+ cfg: cfg,
+ dataStore: dataStore,
+ taskClient: taskClient,
+ triggerClient: triggerClient,
+ runClient: runClient,
+ logStreamer: logStreamer,
+ }
+}
+
+// CreateUploadLocation generates a signed URL for uploading data to the configured storage backend.
+func (s *Service) CreateUploadLocation(
+ ctx context.Context,
+ req *connect.Request[dataproxy.CreateUploadLocationRequest],
+) (*connect.Response[dataproxy.CreateUploadLocationResponse], error) {
+ logger.Infof(ctx, "CreateUploadLocation request for project=%s, domain=%s, filename=%s",
+ req.Msg.Project, req.Msg.Domain, req.Msg.Filename)
+
+ // Validation on request
+ if err := req.Msg.Validate(); err != nil {
+ logger.Errorf(ctx, "Invalid CreateUploadLocation request: %v", err)
+ return nil, connect.NewError(connect.CodeInvalidArgument, err)
+ }
+ if err := validateUploadRequest(ctx, req.Msg, s.cfg); err != nil {
+ logger.Errorf(ctx, "Request validation failed: %v", err)
+ return nil, connect.NewError(connect.CodeInvalidArgument, err)
+ }
+
+ // Build the storage path
+ storagePath, err := s.constructStoragePath(ctx, req.Msg)
+ if err != nil {
+ logger.Errorf(ctx, "Failed to construct storage path: %v", err)
+ return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("failed to construct storage path: %w", err))
+ }
+
+ // Check if file already exists and validate for safe upload
+ if err := s.checkFileExists(ctx, storagePath, req.Msg); err != nil {
+ return nil, err
+ }
+
+ // Set expires_in to default if not provided in request
+ if req.Msg.GetExpiresIn() == nil {
+ req.Msg.ExpiresIn = durationpb.New(s.cfg.Upload.MaxExpiresIn.Duration)
+ }
+
+ // Create signed URL properties
+ expiresIn := req.Msg.GetExpiresIn().AsDuration()
+ props := storage.SignedURLProperties{
+ Scope: stow.ClientMethodPut,
+ ExpiresIn: expiresIn,
+ ContentMD5: base64.StdEncoding.EncodeToString(req.Msg.GetContentMd5()),
+ AddContentMD5Metadata: req.Msg.GetAddContentMd5Metadata(),
+ }
+
+ // Generate signed URL
+ signedResp, err := s.dataStore.CreateSignedURL(ctx, storagePath, props)
+ if err != nil {
+ logger.Errorf(ctx, "Failed to create signed URL: %v", err)
+ return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("failed to create signed URL: %w", err))
+ }
+
+ // Build response
+ expiresAt := time.Now().Add(expiresIn)
+ resp := &dataproxy.CreateUploadLocationResponse{
+ SignedUrl: signedResp.URL.String(),
+ NativeUrl: storagePath.String(),
+ ExpiresAt: timestamppb.New(expiresAt),
+ Headers: signedResp.RequiredRequestHeaders,
+ }
+
+ logger.Infof(ctx, "Successfully created upload location: native_url=%s, expires_at=%s",
+ resp.NativeUrl, resp.ExpiresAt.AsTime().Format(time.RFC3339))
+
+ return connect.NewResponse(resp), nil
+}
+
+// checkFileExists validates whether a file upload is safe by checking existing files.
+// Returns an error if:
+// - File exists without content_md5 provided (cannot verify safe overwrite)
+// - File exists with different content_md5 (prevents accidental overwrite)
+//
+// Returns nil if:
+// - File does not exist (safe to upload)
+// - File exists with matching content_md5 (safe to re-upload same content)
+func (s *Service) checkFileExists(ctx context.Context, storagePath storage.DataReference, req *dataproxy.CreateUploadLocationRequest) error {
+ // Only check if both filename and filename_root are provided
+ if len(req.GetFilename()) == 0 || len(req.GetFilenameRoot()) == 0 {
+ return nil
+ }
+
+ metadata, err := s.dataStore.Head(ctx, storagePath)
+ if err != nil {
+ logger.Errorf(ctx, "Failed to check if file exists at location [%s]: %v", storagePath.String(), err)
+ return connect.NewError(connect.CodeInternal, fmt.Errorf("failed to check if file exists at location [%s]: %w", storagePath.String(), err))
+ }
+
+ if !metadata.Exists() {
+ return nil
+ }
+
+ // Validate based on content hash if file exists
+ // NOTE: This is a best-effort check. Race conditions may occur when multiple clients
+ // upload to the same location simultaneously.
+ if len(req.GetContentMd5()) == 0 {
+ // Cannot verify content, reject to prevent accidental overwrites
+ return connect.NewError(connect.CodeAlreadyExists,
+ fmt.Errorf("file already exists at [%v]; content_md5 is required to verify safe overwrite", storagePath))
+ }
+
+ // Validate hash matches
+ base64Digest := base64.StdEncoding.EncodeToString(req.GetContentMd5())
+ if base64Digest != metadata.ContentMD5() {
+ // Hash mismatch, reject to prevent overwriting different content
+ logger.Errorf(ctx, "File exists at [%v] with different content hash", storagePath)
+ return connect.NewError(connect.CodeAlreadyExists,
+ fmt.Errorf("file already exists at [%v] with different content (hash mismatch)", storagePath))
+ }
+
+ // File exists with matching hash, allow upload to proceed
+ logger.Debugf(ctx, "File already exists at [%v] with matching hash, allowing upload", storagePath)
+ return nil
+}
+
+// constructStoragePath builds the storage path based on the request parameters.
+// Path patterns:
+// - storage_prefix/project/domain/filename_root/filename (if filename_root is provided)
+// - storage_prefix/project/domain/base32_hash/filename (if only content_md5 is provided)
+func (s *Service) constructStoragePath(ctx context.Context, req *dataproxy.CreateUploadLocationRequest) (storage.DataReference, error) {
+ baseRef := s.dataStore.GetBaseContainerFQN(ctx)
+
+ // Build path components: storage_prefix/project/domain/prefix/filename
+ pathComponents := []string{s.cfg.Upload.StoragePrefix, req.GetProject(), req.GetDomain()}
+
+ // Set filename_root or base32-encoded content hash as prefix
+ if len(req.GetFilenameRoot()) > 0 {
+ pathComponents = append(pathComponents, req.GetFilenameRoot())
+ } else {
+ // URL-safe base32 encoding of content hash
+ pathComponents = append(pathComponents, base32.StdEncoding.EncodeToString(req.GetContentMd5()))
+ }
+
+ pathComponents = append(pathComponents, req.GetFilename())
+
+ // Filter out empty components to avoid double slashes in path
+ pathComponents = lo.Filter(pathComponents, func(key string, _ int) bool {
+ return key != ""
+ })
+
+ return s.dataStore.ConstructReference(ctx, baseRef, pathComponents...)
+}
+
+// UploadInputs persists the given inputs to storage and returns a URI and hash
+// that can be passed to CreateRun via OffloadedInputData.
+func (s *Service) UploadInputs(
+ ctx context.Context,
+ req *connect.Request[dataproxy.UploadInputsRequest],
+) (*connect.Response[dataproxy.UploadInputsResponse], error) {
+ logger.Infof(ctx, "UploadInputs request received")
+
+ if err := req.Msg.Validate(); err != nil {
+ logger.Errorf(ctx, "Invalid UploadInputs request: %v", err)
+ return nil, connect.NewError(connect.CodeInvalidArgument, err)
+ }
+
+ // Resolve org/project/domain from the identifier.
+ var org, project, domain string
+ switch id := req.Msg.Id.(type) {
+ case *dataproxy.UploadInputsRequest_RunId:
+ org = id.RunId.GetOrg()
+ project = id.RunId.GetProject()
+ domain = id.RunId.GetDomain()
+ case *dataproxy.UploadInputsRequest_ProjectId:
+ org = id.ProjectId.GetOrganization()
+ project = id.ProjectId.GetName()
+ domain = id.ProjectId.GetDomain()
+ default:
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("id is required"))
+ }
+
+ // Resolve the task template to get cache_ignore_input_vars.
+ taskTemplate, err := s.resolveTaskTemplate(ctx, req.Msg)
+ if err != nil {
+ return nil, err
+ }
+
+ // Filter out cache-ignored inputs before hashing.
+ filteredInputs := filterInputs(req.Msg.GetInputs(), taskTemplate.GetMetadata().GetCacheIgnoreInputVars())
+
+ // Deterministically hash the filtered inputs for cache key computation.
+ inputsHash, err := hashInputsProto(filteredInputs)
+ if err != nil {
+ logger.Errorf(ctx, "Failed to hash inputs: %v", err)
+ return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("failed to hash inputs: %w", err))
+ }
+
+ // Build the storage path: storagePrefix/org/project/domain/offloaded-inputs//inputs.pb
+ storagePrefix := strings.TrimRight(s.cfg.Upload.StoragePrefix, "/")
+ pathComponents := []string{storagePrefix, org, project, domain, "offloaded-inputs", inputsHash}
+ pathComponents = lo.Filter(pathComponents, func(key string, _ int) bool {
+ return key != ""
+ })
+
+ baseRef := s.dataStore.GetBaseContainerFQN(ctx)
+ dirRef, err := s.dataStore.ConstructReference(ctx, baseRef, pathComponents...)
+ if err != nil {
+ logger.Errorf(ctx, "Failed to construct storage path: %v", err)
+ return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("failed to construct storage path: %w", err))
+ }
+
+ inputRef, err := s.dataStore.ConstructReference(ctx, dirRef, "inputs.pb")
+ if err != nil {
+ logger.Errorf(ctx, "Failed to construct input ref: %v", err)
+ return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("failed to construct input ref: %w", err))
+ }
+
+ // Store all inputs (unfiltered) — the hash is over the filtered set for caching.
+ if err := s.dataStore.WriteProtobuf(ctx, inputRef, storage.Options{}, req.Msg.GetInputs()); err != nil {
+ logger.Errorf(ctx, "Failed to write inputs to storage: %v", err)
+ return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("failed to write inputs: %w", err))
+ }
+
+ logger.Infof(ctx, "Successfully uploaded inputs to %s (hash=%s)", inputRef, inputsHash)
+
+ return connect.NewResponse(&dataproxy.UploadInputsResponse{
+ OffloadedInputData: &common.OffloadedInputData{
+ Uri: string(dirRef),
+ InputsHash: inputsHash,
+ },
+ }), nil
+}
+
+// CreateDownloadLink generates signed URL(s) for downloading an artifact associated with a run action.
+func (s *Service) CreateDownloadLink(
+ ctx context.Context,
+ req *connect.Request[dataproxy.CreateDownloadLinkRequest],
+) (*connect.Response[dataproxy.CreateDownloadLinkResponse], error) {
+ logger.Infof(ctx, "CreateDownloadLink request received")
+
+ if err := req.Msg.Validate(); err != nil {
+ logger.Errorf(ctx, "Invalid CreateDownloadLink request: %v", err)
+ return nil, connect.NewError(connect.CodeInvalidArgument, err)
+ }
+
+ if req.Msg.GetArtifactType() != dataproxy.ArtifactType_ARTIFACT_TYPE_REPORT {
+ return nil, connect.NewError(connect.CodeInvalidArgument,
+ fmt.Errorf("artifact_type is required"))
+ }
+
+ // Set expires_in to default if not provided in request
+ if req.Msg.GetExpiresIn() == nil {
+ req.Msg.ExpiresIn = durationpb.New(s.cfg.Download.MaxExpiresIn.Duration)
+ }
+ expiresIn := req.Msg.GetExpiresIn().AsDuration()
+
+ nativeURL, err := s.resolveArtifactURL(ctx, req.Msg)
+ if err != nil {
+ return nil, err
+ }
+
+ ref := storage.DataReference(nativeURL)
+ meta, err := s.dataStore.Head(ctx, ref)
+ if err != nil {
+ logger.Errorf(ctx, "Failed to head artifact at [%s]: %v", nativeURL, err)
+ return nil, connect.NewError(connect.CodeInternal,
+ fmt.Errorf("failed to check artifact existence: %w", err))
+ }
+ if !meta.Exists() {
+ return nil, connect.NewError(connect.CodeNotFound,
+ fmt.Errorf("artifact not found at [%s]", nativeURL))
+ }
+
+ signedResp, err := s.dataStore.CreateSignedURL(ctx, ref, storage.SignedURLProperties{
+ Scope: stow.ClientMethodGet,
+ ExpiresIn: expiresIn,
+ })
+ if err != nil {
+ logger.Errorf(ctx, "Failed to create signed URL for [%s]: %v", nativeURL, err)
+ return nil, connect.NewError(connect.CodeInternal,
+ fmt.Errorf("failed to create signed URL: %w", err))
+ }
+
+ expiresAt := timestamppb.New(time.Now().Add(expiresIn))
+ return connect.NewResponse(&dataproxy.CreateDownloadLinkResponse{
+ PreSignedUrls: &dataproxy.PreSignedURLs{
+ SignedUrl: []string{signedResp.URL.String()},
+ ExpiresAt: expiresAt,
+ },
+ }), nil
+}
+
+// resolveArtifactURL resolves the native storage URL for the requested artifact type and source.
+func (s *Service) resolveArtifactURL(ctx context.Context, req *dataproxy.CreateDownloadLinkRequest) (string, error) {
+ attemptIDEnvelope, ok := req.GetSource().(*dataproxy.CreateDownloadLinkRequest_ActionAttemptId)
+ if !ok {
+ return "", connect.NewError(connect.CodeInvalidArgument,
+ fmt.Errorf("unsupported source type"))
+ }
+
+ attemptID := attemptIDEnvelope.ActionAttemptId
+ actionResp, err := s.runClient.GetActionDetails(ctx, connect.NewRequest(&workflow.GetActionDetailsRequest{
+ ActionId: attemptID.GetActionId(),
+ }))
+ if err != nil {
+ logger.Errorf(ctx, "Failed to get action details for %v: %v", attemptID.GetActionId(), err)
+ return "", connect.NewError(connect.CodeNotFound,
+ fmt.Errorf("failed to get action details: %w", err))
+ }
+
+ // Find the matching attempt by attempt number.
+ var matchedAttempt *workflow.ActionAttempt
+ for _, attempt := range actionResp.Msg.GetDetails().GetAttempts() {
+ if attempt.GetAttempt() == attemptID.GetAttempt() {
+ matchedAttempt = attempt
+ break
+ }
+ }
+ if matchedAttempt == nil {
+ return "", connect.NewError(connect.CodeNotFound,
+ fmt.Errorf("attempt %d not found for action [%v]", attemptID.GetAttempt(), attemptID.GetActionId()))
+ }
+
+ switch req.GetArtifactType() {
+ case dataproxy.ArtifactType_ARTIFACT_TYPE_REPORT:
+ reportURI := matchedAttempt.GetOutputs().GetReportUri()
+ if reportURI == "" {
+ return "", connect.NewError(connect.CodeNotFound,
+ fmt.Errorf("no report URI found for action [%v] attempt %d", attemptID.GetActionId(), attemptID.GetAttempt()))
+ }
+ return reportURI, nil
+ default:
+ return "", connect.NewError(connect.CodeInvalidArgument,
+ fmt.Errorf("unsupported artifact type: %v", req.GetArtifactType()))
+ }
+}
+
+// resolveTaskTemplate resolves the task template from the request's task oneof.
+func (s *Service) resolveTaskTemplate(ctx context.Context, req *dataproxy.UploadInputsRequest) (*flyteIdlCore.TaskTemplate, error) {
+ switch t := req.Task.(type) {
+ case *dataproxy.UploadInputsRequest_TaskSpec:
+ return t.TaskSpec.GetTaskTemplate(), nil
+ case *dataproxy.UploadInputsRequest_TaskId:
+ resp, err := s.taskClient.GetTaskDetails(ctx, connect.NewRequest(&task.GetTaskDetailsRequest{
+ TaskId: t.TaskId,
+ }))
+ if err != nil {
+ logger.Errorf(ctx, "Failed to get task details for %v: %v", t.TaskId, err)
+ return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("failed to get task: %w", err))
+ }
+ return resp.Msg.GetDetails().GetSpec().GetTaskTemplate(), nil
+ case *dataproxy.UploadInputsRequest_TriggerName:
+ triggerResp, err := s.triggerClient.GetTriggerDetails(ctx, connect.NewRequest(&trigger.GetTriggerDetailsRequest{
+ Name: t.TriggerName,
+ }))
+ if err != nil {
+ logger.Errorf(ctx, "Failed to get trigger details for %v: %v", t.TriggerName, err)
+ return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("failed to get trigger: %w", err))
+ }
+ triggerDetails := triggerResp.Msg.GetTrigger()
+ taskID := &task.TaskIdentifier{
+ Org: t.TriggerName.GetOrg(),
+ Project: t.TriggerName.GetProject(),
+ Domain: t.TriggerName.GetDomain(),
+ Name: t.TriggerName.GetTaskName(),
+ Version: triggerDetails.GetSpec().GetTaskVersion(),
+ }
+ taskResp, err := s.taskClient.GetTaskDetails(ctx, connect.NewRequest(&task.GetTaskDetailsRequest{
+ TaskId: taskID,
+ }))
+ if err != nil {
+ logger.Errorf(ctx, "Failed to get task details for trigger %v: %v", t.TriggerName, err)
+ return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("failed to get task for trigger: %w", err))
+ }
+ return taskResp.Msg.GetDetails().GetSpec().GetTaskTemplate(), nil
+ default:
+ return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("task is required"))
+ }
+}
+
+// filterInputs returns a new Inputs with cache-ignored variables removed.
+func filterInputs(inputs *task.Inputs, ignoreVars []string) *task.Inputs {
+ if len(ignoreVars) == 0 {
+ return inputs
+ }
+ var filtered []*task.NamedLiteral
+ for _, nl := range inputs.GetLiterals() {
+ if !slices.Contains(ignoreVars, nl.GetName()) {
+ filtered = append(filtered, nl)
+ }
+ }
+ return &task.Inputs{Literals: filtered}
+}
+
+// GetActionData gets input and output data for an action by calling RunService for URIs
+// and reading the data from storage.
+func (s *Service) GetActionData(
+ ctx context.Context,
+ req *connect.Request[dataproxy.GetActionDataRequest],
+) (*connect.Response[dataproxy.GetActionDataResponse], error) {
+ actionId := req.Msg.GetActionId()
+
+ urisResp, err := s.runClient.GetActionDataURIs(ctx, connect.NewRequest(&workflow.GetActionDataURIsRequest{
+ ActionId: actionId,
+ }))
+ if err != nil {
+ return nil, err
+ }
+
+ resp := &dataproxy.GetActionDataResponse{
+ Inputs: &task.Inputs{},
+ Outputs: &task.Outputs{},
+ }
+
+ group, groupCtx := errgroup.WithContext(ctx)
+
+ if urisResp.Msg.GetInputsUri() != "" {
+ group.Go(func() error {
+ baseRef := storage.DataReference(urisResp.Msg.GetInputsUri())
+ inputRef, err := s.dataStore.ConstructReference(groupCtx, baseRef, "inputs.pb")
+ if err != nil {
+ return connect.NewError(connect.CodeInternal, fmt.Errorf("failed to construct input ref: %w", err))
+ }
+ logger.Infof(groupCtx, "GetActionData: reading inputs from %s", inputRef)
+ if err := s.dataStore.ReadProtobuf(groupCtx, inputRef, resp.Inputs); err != nil {
+ logger.Errorf(groupCtx, "GetActionData: failed to read inputs from %s: %v", inputRef, err)
+ return connect.NewError(connect.CodeInternal, fmt.Errorf("failed to read inputs from %s: %w", inputRef, err))
+ }
+ return nil
+ })
+ }
+
+ if urisResp.Msg.GetOutputsUri() != "" {
+ group.Go(func() error {
+ outputRef := storage.DataReference(urisResp.Msg.GetOutputsUri())
+ logger.Infof(groupCtx, "GetActionData: reading outputs from %s", outputRef)
+ var inputsOrOutputs task.Inputs
+ if err := s.dataStore.ReadProtobuf(groupCtx, outputRef, &inputsOrOutputs); err != nil {
+ logger.Errorf(groupCtx, "GetActionData: failed to read outputs from %s: %v", outputRef, err)
+ return connect.NewError(connect.CodeInternal, fmt.Errorf("failed to read outputs from %s: %w", outputRef, err))
+ }
+ resp.Outputs = &task.Outputs{
+ Literals: inputsOrOutputs.GetLiterals(),
+ }
+ return nil
+ })
+ }
+
+ if err := group.Wait(); err != nil {
+ return nil, err
+ }
+
+ return connect.NewResponse(resp), nil
+}
+
+// TailLogs streams logs for an action attempt.
+func (s *Service) TailLogs(ctx context.Context, req *connect.Request[dataproxy.TailLogsRequest], stream *connect.ServerStream[dataproxy.TailLogsResponse]) error {
+ // Get log context from RunService
+ logCtxResp, err := s.runClient.GetActionLogContext(ctx, connect.NewRequest(&workflow.GetActionLogContextRequest{
+ ActionId: req.Msg.GetActionId(),
+ Attempt: req.Msg.GetAttempt(),
+ }))
+ if err != nil {
+ return err
+ }
+
+ logContext := logCtxResp.Msg.GetLogContext()
+ if logContext == nil {
+ return connect.NewError(connect.CodeNotFound, fmt.Errorf("no log context found"))
+ }
+
+ return s.logStreamer.TailLogs(ctx, logContext, stream)
+}
+
+// hashInputsProto computes a deterministic FNV-64a hash of the serialized inputs.
+func hashInputsProto(inputs proto.Message) (string, error) {
+ marshaller := proto.MarshalOptions{Deterministic: true}
+ data, err := marshaller.Marshal(inputs)
+ if err != nil {
+ return "", fmt.Errorf("failed to marshal inputs: %w", err)
+ }
+ h := fnv.New64a()
+ _, _ = h.Write(data)
+ return base64.RawURLEncoding.EncodeToString(h.Sum(nil)), nil
+}
diff --git a/dataproxy/service/dataproxy_service_test.go b/dataproxy/service/dataproxy_service_test.go
new file mode 100644
index 00000000000..bfba6e48c3f
--- /dev/null
+++ b/dataproxy/service/dataproxy_service_test.go
@@ -0,0 +1,827 @@
+package service
+
+import (
+ "context"
+ "encoding/base64"
+ "net/url"
+ "testing"
+ "time"
+
+ "net/http"
+ "net/http/httptest"
+
+ "connectrpc.com/connect"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/durationpb"
+ "k8s.io/apimachinery/pkg/api/resource"
+
+ "github.com/flyteorg/flyte/v2/dataproxy/config"
+ flyteconfig "github.com/flyteorg/flyte/v2/flytestdlib/config"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ storageMocks "github.com/flyteorg/flyte/v2/flytestdlib/storage/mocks"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/dataproxy"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/dataproxy/dataproxyconnect"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/task"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow"
+ workflowMocks "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow/workflowconnect/mocks"
+)
+
+func TestCreateUploadLocation(t *testing.T) {
+ ctx := context.Background()
+
+ cfg := config.DataProxyConfig{
+ Upload: config.DataProxyUploadConfig{
+ MaxExpiresIn: flyteconfig.Duration{Duration: 1 * time.Hour},
+ MaxSize: resource.MustParse("100Mi"), // 100MB
+ StoragePrefix: "uploads",
+ DefaultFileNameLength: 20,
+ },
+ }
+
+ tests := []struct {
+ name string
+ req *dataproxy.CreateUploadLocationRequest
+ wantErr bool
+ errContains string
+ validateResult func(t *testing.T, resp *connect.Response[dataproxy.CreateUploadLocationResponse])
+ }{
+ {
+ name: "success with valid request",
+ req: &dataproxy.CreateUploadLocationRequest{
+ Project: "test-project",
+ Domain: "test-domain",
+ Filename: "test-file.txt",
+ FilenameRoot: "test-root",
+ ContentMd5: []byte("test-hash"),
+ ExpiresIn: durationpb.New(30 * time.Minute),
+ },
+ wantErr: false,
+ validateResult: func(t *testing.T, resp *connect.Response[dataproxy.CreateUploadLocationResponse]) {
+ assert.Contains(t, resp.Msg.SignedUrl, "https://test-bucket")
+ assert.Contains(t, resp.Msg.NativeUrl, "uploads/test-project/test-domain/test-root/test-file.txt")
+ },
+ },
+ {
+ name: "validation error - missing both filename_root and content_md5",
+ req: &dataproxy.CreateUploadLocationRequest{
+ Project: "test-project",
+ Domain: "test-domain",
+ Filename: "test-file.txt",
+ },
+ wantErr: true,
+ errContains: "either filename_root or content_md5 must be provided",
+ },
+ {
+ name: "validation error - expires_in exceeds maximum",
+ req: &dataproxy.CreateUploadLocationRequest{
+ Project: "test-project",
+ Domain: "test-domain",
+ Filename: "test-file.txt",
+ FilenameRoot: "test-root",
+ ExpiresIn: durationpb.New(2 * time.Hour),
+ },
+ wantErr: true,
+ errContains: "exceeds maximum allowed duration",
+ },
+ {
+ name: "validation error - content_length exceeds maximum",
+ req: &dataproxy.CreateUploadLocationRequest{
+ Project: "test-project",
+ Domain: "test-domain",
+ Filename: "test-file.txt",
+ FilenameRoot: "test-root",
+ ContentLength: 1024 * 1024 * 200, // 200MB
+ },
+ wantErr: true,
+ errContains: "exceeds maximum allowed size",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ mockStore := setupMockDataStore(t)
+ service := NewService(cfg, mockStore, nil, nil, nil, nil)
+
+ req := &connect.Request[dataproxy.CreateUploadLocationRequest]{
+ Msg: tt.req,
+ }
+
+ resp, err := service.CreateUploadLocation(ctx, req)
+
+ if tt.wantErr {
+ assert.Error(t, err)
+ assert.Nil(t, resp)
+ assert.Contains(t, err.Error(), tt.errContains)
+ } else {
+ assert.NoError(t, err)
+ assert.NotNil(t, resp)
+ if tt.validateResult != nil {
+ tt.validateResult(t, resp)
+ }
+ }
+ })
+ }
+}
+
+func TestCheckFileExists(t *testing.T) {
+ ctx := context.Background()
+
+ cfg := config.DataProxyConfig{
+ Upload: config.DataProxyUploadConfig{
+ StoragePrefix: "uploads",
+ },
+ }
+
+ tests := []struct {
+ name string
+ req *dataproxy.CreateUploadLocationRequest
+ existingFileMD5 string // Empty means file doesn't exist
+ expectErr bool
+ errContains string
+ }{
+ {
+ name: "file does not exist",
+ req: &dataproxy.CreateUploadLocationRequest{
+ Project: "test-project",
+ Domain: "test-domain",
+ Filename: "test-file.txt",
+ FilenameRoot: "test-root",
+ },
+ existingFileMD5: "", // File doesn't exist
+ expectErr: false,
+ },
+ {
+ name: "file exists without hash provided",
+ req: &dataproxy.CreateUploadLocationRequest{
+ Project: "test-project",
+ Domain: "test-domain",
+ Filename: "test-file.txt",
+ FilenameRoot: "test-root",
+ // No ContentMd5 provided
+ },
+ existingFileMD5: "existing-hash",
+ expectErr: true,
+ errContains: "content_md5 is required to verify safe overwrite",
+ },
+ {
+ name: "file exists with matching hash",
+ req: &dataproxy.CreateUploadLocationRequest{
+ Project: "test-project",
+ Domain: "test-domain",
+ Filename: "test-file.txt",
+ FilenameRoot: "test-root",
+ ContentMd5: []byte("test-hash-123"),
+ },
+ existingFileMD5: base64.StdEncoding.EncodeToString([]byte("test-hash-123")),
+ expectErr: false,
+ },
+ {
+ name: "file exists with different hash",
+ req: &dataproxy.CreateUploadLocationRequest{
+ Project: "test-project",
+ Domain: "test-domain",
+ Filename: "test-file.txt",
+ FilenameRoot: "test-root",
+ ContentMd5: []byte("different-hash"),
+ },
+ existingFileMD5: base64.StdEncoding.EncodeToString([]byte("existing-hash")),
+ expectErr: true,
+ errContains: "with different content (hash mismatch)",
+ },
+ {
+ name: "skip check when filename is empty",
+ req: &dataproxy.CreateUploadLocationRequest{
+ Project: "test-project",
+ Domain: "test-domain",
+ Filename: "", // Empty filename
+ FilenameRoot: "test-root",
+ },
+ existingFileMD5: "",
+ expectErr: false,
+ },
+ {
+ name: "skip check when filename_root is empty",
+ req: &dataproxy.CreateUploadLocationRequest{
+ Project: "test-project",
+ Domain: "test-domain",
+ Filename: "test-file.txt",
+ FilenameRoot: "", // Empty filename_root
+ },
+ existingFileMD5: "",
+ expectErr: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ var mockStore *storage.DataStore
+ if tt.existingFileMD5 == "" {
+ mockStore = setupMockDataStore(t)
+ } else {
+ mockStore = setupMockDataStoreWithExistingFile(t, tt.existingFileMD5)
+ }
+
+ service := NewService(cfg, mockStore, nil, nil, nil, nil)
+ storagePath := storage.DataReference("s3://test-bucket/uploads/test-project/test-domain/test-root/test-file.txt")
+
+ err := service.checkFileExists(ctx, storagePath, tt.req)
+
+ if tt.expectErr {
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), tt.errContains)
+ } else {
+ assert.NoError(t, err)
+ }
+ })
+ }
+}
+
+func TestConstructStoragePath(t *testing.T) {
+ ctx := context.Background()
+
+ cfg := config.DataProxyConfig{
+ Upload: config.DataProxyUploadConfig{
+ StoragePrefix: "uploads",
+ },
+ }
+
+ tests := []struct {
+ name string
+ req *dataproxy.CreateUploadLocationRequest
+ expectedPath string
+ }{
+ {
+ name: "with filename_root",
+ req: &dataproxy.CreateUploadLocationRequest{
+ Org: "test-org",
+ Project: "test-project",
+ Domain: "test-domain",
+ Filename: "test-file.txt",
+ FilenameRoot: "test-root",
+ },
+ expectedPath: "s3://test-bucket/uploads/test-project/test-domain/test-root/test-file.txt",
+ },
+ {
+ name: "with content_md5 uses base32 encoding",
+ req: &dataproxy.CreateUploadLocationRequest{
+ Project: "test-project",
+ Domain: "test-domain",
+ Filename: "test-file.txt",
+ ContentMd5: []byte("test-hash"),
+ },
+ // base32 encoding for "test-hash" is ORSXG5BNNBQXG2A=
+ expectedPath: "s3://test-bucket/uploads/test-project/test-domain/ORSXG5BNNBQXG2A=/test-file.txt",
+ },
+ {
+ name: "filters empty org component",
+ req: &dataproxy.CreateUploadLocationRequest{
+ Org: "", // Empty org
+ Project: "test-project",
+ Domain: "test-domain",
+ Filename: "test-file.txt",
+ FilenameRoot: "test-root",
+ },
+ expectedPath: "s3://test-bucket/uploads/test-project/test-domain/test-root/test-file.txt",
+ },
+ {
+ name: "with all components including org",
+ req: &dataproxy.CreateUploadLocationRequest{
+ Org: "test-org",
+ Project: "test-project",
+ Domain: "test-domain",
+ Filename: "test-file.txt",
+ FilenameRoot: "test-root",
+ },
+ expectedPath: "s3://test-bucket/uploads/test-project/test-domain/test-root/test-file.txt",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ mockStore := setupMockDataStore(t)
+ service := NewService(cfg, mockStore, nil, nil, nil, nil)
+
+ path, err := service.constructStoragePath(ctx, tt.req)
+
+ assert.NoError(t, err)
+ assert.Equal(t, tt.expectedPath, path.String())
+ })
+ }
+}
+
+// Helper functions to setup mocks
+
+// simpleRefConstructor is a simple implementation of ReferenceConstructor for testing
+type simpleRefConstructor struct{}
+
+func (s *simpleRefConstructor) ConstructReference(ctx context.Context, base storage.DataReference, keys ...string) (storage.DataReference, error) {
+ path := string(base)
+ for _, key := range keys {
+ if key != "" {
+ path += "/" + key
+ }
+ }
+ return storage.DataReference(path), nil
+}
+
+func setupMockDataStore(t *testing.T) *storage.DataStore {
+ mockComposedStore := storageMocks.NewComposedProtobufStore(t)
+
+ // Setup base container
+ mockComposedStore.On("GetBaseContainerFQN", mock.Anything).Return(storage.DataReference("s3://test-bucket")).Maybe()
+
+ // Setup Head to return file does not exist
+ mockMetadata := storageMocks.NewMetadata(t)
+ mockMetadata.On("Exists").Return(false).Maybe()
+ mockComposedStore.On("Head", mock.Anything, mock.Anything).Return(mockMetadata, nil).Maybe()
+
+ // Setup CreateSignedURL
+ testURL, _ := url.Parse("https://test-bucket.s3.amazonaws.com/signed-url")
+ mockComposedStore.On("CreateSignedURL", mock.Anything, mock.Anything, mock.Anything).Return(
+ storage.SignedURLResponse{
+ URL: *testURL,
+ RequiredRequestHeaders: map[string]string{"Content-Type": "application/octet-stream"},
+ }, nil).Maybe()
+
+ return &storage.DataStore{
+ ComposedProtobufStore: mockComposedStore,
+ ReferenceConstructor: &simpleRefConstructor{},
+ }
+}
+
+func TestUploadInputs(t *testing.T) {
+ ctx := context.Background()
+
+ cfg := config.DataProxyConfig{
+ Upload: config.DataProxyUploadConfig{
+ StoragePrefix: "uploads",
+ },
+ }
+
+ testTaskSpec := &task.TaskSpec{
+ TaskTemplate: &core.TaskTemplate{
+ Id: &core.Identifier{Name: "test-task"},
+ Metadata: &core.TaskMetadata{},
+ },
+ }
+
+ testTaskSpecWithIgnoredVars := &task.TaskSpec{
+ TaskTemplate: &core.TaskTemplate{
+ Id: &core.Identifier{Name: "test-task"},
+ Metadata: &core.TaskMetadata{
+ CacheIgnoreInputVars: []string{"y"},
+ },
+ },
+ }
+
+ tests := []struct {
+ name string
+ req *dataproxy.UploadInputsRequest
+ wantErr bool
+ errCode connect.Code
+ validateResult func(t *testing.T, resp *connect.Response[dataproxy.UploadInputsResponse])
+ }{
+ {
+ name: "success with run_id and task_spec",
+ req: &dataproxy.UploadInputsRequest{
+ Id: &dataproxy.UploadInputsRequest_RunId{
+ RunId: &common.RunIdentifier{
+ Org: "test-org",
+ Project: "test-project",
+ Domain: "test-domain",
+ Name: "test-run",
+ },
+ },
+ Task: &dataproxy.UploadInputsRequest_TaskSpec{TaskSpec: testTaskSpec},
+ Inputs: &task.Inputs{
+ Literals: []*task.NamedLiteral{
+ {Name: "x", Value: &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_Integer{Integer: 42}}}}}}},
+ },
+ },
+ },
+ wantErr: false,
+ validateResult: func(t *testing.T, resp *connect.Response[dataproxy.UploadInputsResponse]) {
+ assert.NotNil(t, resp.Msg.OffloadedInputData)
+ assert.NotEmpty(t, resp.Msg.OffloadedInputData.Uri)
+ assert.NotEmpty(t, resp.Msg.OffloadedInputData.InputsHash)
+ assert.Contains(t, resp.Msg.OffloadedInputData.Uri, "uploads/test-org/test-project/test-domain/offloaded-inputs/")
+ },
+ },
+ {
+ name: "success with project_id",
+ req: &dataproxy.UploadInputsRequest{
+ Id: &dataproxy.UploadInputsRequest_ProjectId{
+ ProjectId: &common.ProjectIdentifier{
+ Organization: "test-org",
+ Name: "test-project",
+ Domain: "test-domain",
+ },
+ },
+ Task: &dataproxy.UploadInputsRequest_TaskSpec{TaskSpec: testTaskSpec},
+ Inputs: &task.Inputs{
+ Literals: []*task.NamedLiteral{
+ {Name: "y", Value: &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "hello"}}}}}}},
+ },
+ },
+ },
+ wantErr: false,
+ validateResult: func(t *testing.T, resp *connect.Response[dataproxy.UploadInputsResponse]) {
+ assert.NotNil(t, resp.Msg.OffloadedInputData)
+ assert.Contains(t, resp.Msg.OffloadedInputData.Uri, "uploads/test-org/test-project/test-domain/offloaded-inputs/")
+ },
+ },
+ {
+ name: "cache_ignore_input_vars excludes inputs from hash",
+ req: &dataproxy.UploadInputsRequest{
+ Id: &dataproxy.UploadInputsRequest_RunId{
+ RunId: &common.RunIdentifier{
+ Org: "org", Project: "proj", Domain: "dom", Name: "run1",
+ },
+ },
+ Task: &dataproxy.UploadInputsRequest_TaskSpec{TaskSpec: testTaskSpecWithIgnoredVars},
+ Inputs: &task.Inputs{
+ Literals: []*task.NamedLiteral{
+ {Name: "x", Value: &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_Integer{Integer: 1}}}}}}},
+ {Name: "y", Value: &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_Integer{Integer: 2}}}}}}},
+ },
+ },
+ },
+ wantErr: false,
+ validateResult: func(t *testing.T, resp *connect.Response[dataproxy.UploadInputsResponse]) {
+ assert.NotEmpty(t, resp.Msg.OffloadedInputData.InputsHash)
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ mockStore := setupMockDataStoreWithWriteProtobuf(t)
+ svc := NewService(cfg, mockStore, nil, nil, nil, nil)
+
+ req := &connect.Request[dataproxy.UploadInputsRequest]{
+ Msg: tt.req,
+ }
+
+ resp, err := svc.UploadInputs(ctx, req)
+
+ if tt.wantErr {
+ assert.Error(t, err)
+ assert.Nil(t, resp)
+ if tt.errCode != 0 {
+ assert.Equal(t, tt.errCode, connect.CodeOf(err))
+ }
+ } else {
+ assert.NoError(t, err)
+ assert.NotNil(t, resp)
+ if tt.validateResult != nil {
+ tt.validateResult(t, resp)
+ }
+ }
+ })
+ }
+}
+
+func setupMockDataStoreWithWriteProtobuf(t *testing.T) *storage.DataStore {
+ mockComposedStore := storageMocks.NewComposedProtobufStore(t)
+
+ mockComposedStore.On("GetBaseContainerFQN", mock.Anything).Return(storage.DataReference("s3://test-bucket")).Maybe()
+ mockComposedStore.On("WriteProtobuf", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe()
+
+ return &storage.DataStore{
+ ComposedProtobufStore: mockComposedStore,
+ ReferenceConstructor: &simpleRefConstructor{},
+ }
+}
+
+func TestGetActionData(t *testing.T) {
+ ctx := context.Background()
+ cfg := config.DataProxyConfig{}
+
+ actionID := &common.ActionIdentifier{
+ Name: "a0",
+ Run: &common.RunIdentifier{
+ Org: "org", Project: "proj", Domain: "dom", Name: "run1",
+ },
+ }
+
+ storedInputs := &task.Inputs{
+ Literals: []*task.NamedLiteral{
+ {Name: "x", Value: &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_Integer{Integer: 1}}}}}}},
+ },
+ }
+ storedOutputs := &task.Inputs{
+ Literals: []*task.NamedLiteral{
+ {Name: "o", Value: &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "result"}}}}}}},
+ },
+ }
+
+ tests := []struct {
+ name string
+ inputsURI string
+ outputsURI string
+ runClientErr error
+ readInputsErr error
+ readOutputsErr error
+ wantErr bool
+ expectInputsLen int
+ expectOutputsLen int
+ }{
+ {
+ name: "success with both inputs and outputs",
+ inputsURI: "s3://test-bucket/inputs-dir",
+ outputsURI: "s3://test-bucket/outputs/outputs.pb",
+ expectInputsLen: 1,
+ expectOutputsLen: 1,
+ },
+ {
+ name: "success with only inputs",
+ inputsURI: "s3://test-bucket/inputs-dir",
+ outputsURI: "",
+ expectInputsLen: 1,
+ expectOutputsLen: 0,
+ },
+ {
+ name: "success with only outputs",
+ inputsURI: "",
+ outputsURI: "s3://test-bucket/outputs/outputs.pb",
+ expectInputsLen: 0,
+ expectOutputsLen: 1,
+ },
+ {
+ name: "success with neither inputs nor outputs",
+ inputsURI: "",
+ outputsURI: "",
+ expectInputsLen: 0,
+ expectOutputsLen: 0,
+ },
+ {
+ name: "RunService error propagates",
+ runClientErr: connect.NewError(connect.CodeNotFound, assertErr("not found")),
+ wantErr: true,
+ },
+ {
+ name: "read inputs error propagates",
+ inputsURI: "s3://test-bucket/inputs-dir",
+ readInputsErr: assertErr("read failed"),
+ wantErr: true,
+ },
+ {
+ name: "read outputs error propagates",
+ outputsURI: "s3://test-bucket/outputs/outputs.pb",
+ readOutputsErr: assertErr("read failed"),
+ wantErr: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ runClient := workflowMocks.NewRunServiceClient(t)
+ if tt.runClientErr != nil {
+ runClient.EXPECT().GetActionDataURIs(mock.Anything, mock.Anything).Return(nil, tt.runClientErr)
+ } else {
+ runClient.EXPECT().GetActionDataURIs(mock.Anything, mock.Anything).Return(
+ connect.NewResponse(&workflow.GetActionDataURIsResponse{
+ InputsUri: tt.inputsURI,
+ OutputsUri: tt.outputsURI,
+ }), nil)
+ }
+
+ mockComposedStore := storageMocks.NewComposedProtobufStore(t)
+
+ if tt.inputsURI != "" {
+ expectedInputRef := storage.DataReference(tt.inputsURI + "/inputs.pb")
+ call := mockComposedStore.On("ReadProtobuf", mock.Anything, expectedInputRef, mock.Anything)
+ if tt.readInputsErr != nil {
+ call.Return(tt.readInputsErr).Maybe()
+ } else {
+ call.Run(func(args mock.Arguments) {
+ msg := args.Get(2).(proto.Message)
+ proto.Reset(msg)
+ proto.Merge(msg, storedInputs)
+ }).Return(nil).Maybe()
+ }
+ }
+
+ if tt.outputsURI != "" {
+ expectedOutputRef := storage.DataReference(tt.outputsURI)
+ call := mockComposedStore.On("ReadProtobuf", mock.Anything, expectedOutputRef, mock.Anything)
+ if tt.readOutputsErr != nil {
+ call.Return(tt.readOutputsErr).Maybe()
+ } else {
+ call.Run(func(args mock.Arguments) {
+ msg := args.Get(2).(proto.Message)
+ proto.Reset(msg)
+ proto.Merge(msg, storedOutputs)
+ }).Return(nil).Maybe()
+ }
+ }
+
+ ds := &storage.DataStore{
+ ComposedProtobufStore: mockComposedStore,
+ ReferenceConstructor: &simpleRefConstructor{},
+ }
+ svc := NewService(cfg, ds, nil, nil, runClient, nil)
+
+ resp, err := svc.GetActionData(ctx, connect.NewRequest(&dataproxy.GetActionDataRequest{
+ ActionId: actionID,
+ }))
+
+ if tt.wantErr {
+ assert.Error(t, err)
+ assert.Nil(t, resp)
+ return
+ }
+ assert.NoError(t, err)
+ assert.NotNil(t, resp)
+ assert.Len(t, resp.Msg.GetInputs().GetLiterals(), tt.expectInputsLen)
+ assert.Len(t, resp.Msg.GetOutputs().GetLiterals(), tt.expectOutputsLen)
+ if tt.expectInputsLen > 0 {
+ assert.Equal(t, "x", resp.Msg.GetInputs().GetLiterals()[0].GetName())
+ }
+ if tt.expectOutputsLen > 0 {
+ assert.Equal(t, "o", resp.Msg.GetOutputs().GetLiterals()[0].GetName())
+ }
+ })
+ }
+}
+
+type assertErr string
+
+func (e assertErr) Error() string { return string(e) }
+
+func setupMockDataStoreWithExistingFile(t *testing.T, contentMD5 string) *storage.DataStore {
+ mockComposedStore := storageMocks.NewComposedProtobufStore(t)
+
+ // Setup base container
+ mockComposedStore.On("GetBaseContainerFQN", mock.Anything).Return(storage.DataReference("s3://test-bucket")).Maybe()
+
+ // Setup Head to return file exists with given hash
+ mockMetadata := storageMocks.NewMetadata(t)
+ mockMetadata.On("Exists").Return(true)
+ mockMetadata.On("ContentMD5").Return(contentMD5).Maybe()
+ mockComposedStore.On("Head", mock.Anything, mock.Anything).Return(mockMetadata, nil)
+
+ return &storage.DataStore{
+ ComposedProtobufStore: mockComposedStore,
+ ReferenceConstructor: &simpleRefConstructor{},
+ }
+}
+
+// mockLogStreamer implements logs.LogStreamer for tests.
+type mockLogStreamer struct {
+ mock.Mock
+}
+
+func (m *mockLogStreamer) TailLogs(ctx context.Context, logContext *core.LogContext, stream *connect.ServerStream[dataproxy.TailLogsResponse]) error {
+ args := m.Called(ctx, logContext, stream)
+ return args.Error(0)
+}
+
+func newTailLogsTestClient(t *testing.T, svc *Service) dataproxyconnect.DataProxyServiceClient {
+ path, handler := dataproxyconnect.NewDataProxyServiceHandler(svc)
+ mux := http.NewServeMux()
+ mux.Handle(path, handler)
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+ return dataproxyconnect.NewDataProxyServiceClient(http.DefaultClient, server.URL)
+}
+
+func TestTailLogs(t *testing.T) {
+ actionID := &common.ActionIdentifier{
+ Run: &common.RunIdentifier{
+ Org: "test-org", Project: "test-project", Domain: "test-domain", Name: "rtest12345",
+ },
+ Name: "a0",
+ }
+
+ logContext := &core.LogContext{
+ PrimaryPodName: "my-pod",
+ Pods: []*core.PodLogContext{
+ {PodName: "my-pod", Namespace: "ns"},
+ },
+ }
+
+ t.Run("happy path streams a message", func(t *testing.T) {
+ runClient := workflowMocks.NewRunServiceClient(t)
+ runClient.EXPECT().GetActionLogContext(mock.Anything, mock.Anything).Return(
+ connect.NewResponse(&workflow.GetActionLogContextResponse{
+ LogContext: logContext,
+ Cluster: "c1",
+ }), nil)
+
+ streamer := &mockLogStreamer{}
+ streamer.On("TailLogs", mock.Anything, logContext, mock.Anything).Run(func(args mock.Arguments) {
+ stream := args.Get(2).(*connect.ServerStream[dataproxy.TailLogsResponse])
+ _ = stream.Send(&dataproxy.TailLogsResponse{})
+ }).Return(nil)
+
+ svc := NewService(config.DataProxyConfig{}, nil, nil, nil, runClient, streamer)
+ client := newTailLogsTestClient(t, svc)
+
+ stream, err := client.TailLogs(context.Background(), connect.NewRequest(&dataproxy.TailLogsRequest{
+ ActionId: actionID,
+ Attempt: 1,
+ }))
+ assert.NoError(t, err)
+
+ assert.True(t, stream.Receive())
+ assert.NotNil(t, stream.Msg())
+ assert.False(t, stream.Receive())
+ assert.NoError(t, stream.Err())
+
+ streamer.AssertExpectations(t)
+ })
+
+ t.Run("GetActionLogContext error propagates", func(t *testing.T) {
+ runClient := workflowMocks.NewRunServiceClient(t)
+ runClient.EXPECT().GetActionLogContext(mock.Anything, mock.Anything).Return(
+ nil, connect.NewError(connect.CodeNotFound, assertErr("action missing")))
+
+ streamer := &mockLogStreamer{}
+ svc := NewService(config.DataProxyConfig{}, nil, nil, nil, runClient, streamer)
+ client := newTailLogsTestClient(t, svc)
+
+ stream, err := client.TailLogs(context.Background(), connect.NewRequest(&dataproxy.TailLogsRequest{
+ ActionId: actionID,
+ Attempt: 1,
+ }))
+ assert.NoError(t, err)
+ assert.False(t, stream.Receive())
+ assert.Error(t, stream.Err())
+ assert.Equal(t, connect.CodeNotFound, connect.CodeOf(stream.Err()))
+
+ streamer.AssertNotCalled(t, "TailLogs", mock.Anything, mock.Anything, mock.Anything)
+ })
+
+ t.Run("nil log context returns NotFound", func(t *testing.T) {
+ runClient := workflowMocks.NewRunServiceClient(t)
+ runClient.EXPECT().GetActionLogContext(mock.Anything, mock.Anything).Return(
+ connect.NewResponse(&workflow.GetActionLogContextResponse{
+ LogContext: nil,
+ }), nil)
+
+ streamer := &mockLogStreamer{}
+ svc := NewService(config.DataProxyConfig{}, nil, nil, nil, runClient, streamer)
+ client := newTailLogsTestClient(t, svc)
+
+ stream, err := client.TailLogs(context.Background(), connect.NewRequest(&dataproxy.TailLogsRequest{
+ ActionId: actionID,
+ Attempt: 1,
+ }))
+ assert.NoError(t, err)
+ assert.False(t, stream.Receive())
+ assert.Error(t, stream.Err())
+ assert.Equal(t, connect.CodeNotFound, connect.CodeOf(stream.Err()))
+
+ streamer.AssertNotCalled(t, "TailLogs", mock.Anything, mock.Anything, mock.Anything)
+ })
+
+ t.Run("streamer error propagates", func(t *testing.T) {
+ runClient := workflowMocks.NewRunServiceClient(t)
+ runClient.EXPECT().GetActionLogContext(mock.Anything, mock.Anything).Return(
+ connect.NewResponse(&workflow.GetActionLogContextResponse{LogContext: logContext}), nil)
+
+ streamer := &mockLogStreamer{}
+ streamer.On("TailLogs", mock.Anything, logContext, mock.Anything).Return(
+ connect.NewError(connect.CodeInternal, assertErr("streamer boom")))
+
+ svc := NewService(config.DataProxyConfig{}, nil, nil, nil, runClient, streamer)
+ client := newTailLogsTestClient(t, svc)
+
+ stream, err := client.TailLogs(context.Background(), connect.NewRequest(&dataproxy.TailLogsRequest{
+ ActionId: actionID,
+ Attempt: 1,
+ }))
+ assert.NoError(t, err)
+ assert.False(t, stream.Receive())
+ assert.Error(t, stream.Err())
+ assert.Equal(t, connect.CodeInternal, connect.CodeOf(stream.Err()))
+
+ streamer.AssertExpectations(t)
+ })
+
+ t.Run("passes action_id and attempt to RunService", func(t *testing.T) {
+ runClient := workflowMocks.NewRunServiceClient(t)
+ runClient.EXPECT().GetActionLogContext(mock.Anything, mock.MatchedBy(func(r *connect.Request[workflow.GetActionLogContextRequest]) bool {
+ return proto.Equal(r.Msg.GetActionId(), actionID) && r.Msg.GetAttempt() == 3
+ })).Return(connect.NewResponse(&workflow.GetActionLogContextResponse{LogContext: logContext}), nil)
+
+ streamer := &mockLogStreamer{}
+ streamer.On("TailLogs", mock.Anything, logContext, mock.Anything).Return(nil)
+
+ svc := NewService(config.DataProxyConfig{}, nil, nil, nil, runClient, streamer)
+ client := newTailLogsTestClient(t, svc)
+
+ stream, err := client.TailLogs(context.Background(), connect.NewRequest(&dataproxy.TailLogsRequest{
+ ActionId: actionID,
+ Attempt: 3,
+ }))
+ assert.NoError(t, err)
+ assert.False(t, stream.Receive())
+ assert.NoError(t, stream.Err())
+ })
+}
diff --git a/dataproxy/service/validation.go b/dataproxy/service/validation.go
new file mode 100644
index 00000000000..e15527706c5
--- /dev/null
+++ b/dataproxy/service/validation.go
@@ -0,0 +1,39 @@
+package service
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/flyteorg/flyte/v2/dataproxy/config"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/dataproxy"
+)
+
+// validateUploadRequest performs validation on the upload request.
+func validateUploadRequest(ctx context.Context, req *dataproxy.CreateUploadLocationRequest, cfg config.DataProxyConfig) error {
+ if len(req.FilenameRoot) == 0 && len(req.ContentMd5) == 0 {
+ return fmt.Errorf("either filename_root or content_md5 must be provided")
+ }
+
+ // Validate expires_in against platform maximum
+ if req.ExpiresIn != nil {
+ if !req.ExpiresIn.IsValid() {
+ return fmt.Errorf("expires_in (%v) is invalid", req.ExpiresIn)
+ }
+ maxExpiration := cfg.Upload.MaxExpiresIn.Duration
+ if maxExpiration > 0 && req.ExpiresIn.AsDuration() > maxExpiration {
+ return fmt.Errorf("expires_in (%v) exceeds maximum allowed duration of %v",
+ req.ExpiresIn.AsDuration(), maxExpiration)
+ }
+ }
+
+ // Validate content_length against platform maximum
+ if req.ContentLength > 0 {
+ maxSize := cfg.Upload.MaxSize.Value()
+ if maxSize > 0 && req.ContentLength > maxSize {
+ return fmt.Errorf("content_length (%d bytes) exceeds maximum allowed size of %d bytes",
+ req.ContentLength, maxSize)
+ }
+ }
+
+ return nil
+}
diff --git a/dataproxy/service/validation_test.go b/dataproxy/service/validation_test.go
new file mode 100644
index 00000000000..058864828c5
--- /dev/null
+++ b/dataproxy/service/validation_test.go
@@ -0,0 +1,95 @@
+package service
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "google.golang.org/protobuf/types/known/durationpb"
+ "k8s.io/apimachinery/pkg/api/resource"
+
+ "github.com/flyteorg/flyte/v2/dataproxy/config"
+ flyteconfig "github.com/flyteorg/flyte/v2/flytestdlib/config"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/dataproxy"
+)
+
+func TestValidateUploadRequest(t *testing.T) {
+ ctx := context.Background()
+ cfg := config.DataProxyConfig{
+ Upload: config.DataProxyUploadConfig{
+ MaxExpiresIn: flyteconfig.Duration{Duration: 1 * time.Hour},
+ MaxSize: resource.MustParse("100Mi"),
+ },
+ }
+
+ tests := []struct {
+ name string
+ req *dataproxy.CreateUploadLocationRequest
+ wantErr bool
+ errMsg string
+ }{
+ {
+ name: "valid request with filename_root",
+ req: &dataproxy.CreateUploadLocationRequest{
+ FilenameRoot: "test-root",
+ ExpiresIn: durationpb.New(30 * time.Minute),
+ },
+ wantErr: false,
+ },
+ {
+ name: "valid request with content_md5",
+ req: &dataproxy.CreateUploadLocationRequest{
+ ContentMd5: []byte("test-hash"),
+ ExpiresIn: durationpb.New(30 * time.Minute),
+ },
+ wantErr: false,
+ },
+ {
+ name: "missing both filename_root and content_md5",
+ req: &dataproxy.CreateUploadLocationRequest{
+ ExpiresIn: durationpb.New(30 * time.Minute),
+ },
+ wantErr: true,
+ errMsg: "either filename_root or content_md5 must be provided",
+ },
+ {
+ name: "expires_in exceeds max",
+ req: &dataproxy.CreateUploadLocationRequest{
+ FilenameRoot: "test-root",
+ ExpiresIn: durationpb.New(2 * time.Hour),
+ },
+ wantErr: true,
+ errMsg: "exceeds maximum allowed duration",
+ },
+ {
+ name: "content_length exceeds max",
+ req: &dataproxy.CreateUploadLocationRequest{
+ FilenameRoot: "test-root",
+ ContentLength: 1024 * 1024 * 200,
+ },
+ wantErr: true,
+ errMsg: "exceeds maximum allowed size",
+ },
+ {
+ name: "valid request without expires_in (will use default)",
+ req: &dataproxy.CreateUploadLocationRequest{
+ FilenameRoot: "test-root",
+ },
+ wantErr: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ err := validateUploadRequest(ctx, tt.req, cfg)
+
+ if tt.wantErr {
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), tt.errMsg)
+ } else {
+ assert.NoError(t, err)
+ }
+ })
+ }
+}
diff --git a/dataproxy/setup.go b/dataproxy/setup.go
new file mode 100644
index 00000000000..adbe891a17d
--- /dev/null
+++ b/dataproxy/setup.go
@@ -0,0 +1,60 @@
+package dataproxy
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow/workflowconnect"
+
+ "github.com/flyteorg/flyte/v2/dataproxy/config"
+ "github.com/flyteorg/flyte/v2/dataproxy/logs"
+ "github.com/flyteorg/flyte/v2/dataproxy/service"
+ "github.com/flyteorg/flyte/v2/flytestdlib/app"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/cluster/clusterconnect"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/dataproxy/dataproxyconnect"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/task/taskconnect"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/trigger/triggerconnect"
+)
+
+// Setup registers the DataProxy service handler on the SetupContext mux.
+// Requires sc.DataStore to be set.
+func Setup(ctx context.Context, sc *app.SetupContext) error {
+ cfg := config.GetConfig()
+
+ baseURL := sc.BaseURL
+ taskClient := taskconnect.NewTaskServiceClient(http.DefaultClient, baseURL)
+ triggerClient := triggerconnect.NewTriggerServiceClient(http.DefaultClient, baseURL)
+ runClient := workflowconnect.NewRunServiceClient(http.DefaultClient, baseURL)
+
+ var logStreamer logs.LogStreamer
+ if sc.K8sConfig != nil {
+ var err error
+ logStreamer, err = logs.NewK8sLogStreamer(sc.K8sConfig)
+ if err != nil {
+ return fmt.Errorf("failed to create k8s log streamer: %w", err)
+ }
+ }
+
+ svc := service.NewService(*cfg, sc.DataStore, taskClient, triggerClient, runClient, logStreamer)
+
+ path, handler := dataproxyconnect.NewDataProxyServiceHandler(svc)
+ sc.Mux.Handle(path, handler)
+ logger.Infof(ctx, "Mounted DataProxyService at %s", path)
+
+ clusterSvc := service.NewClusterService()
+ clusterPath, clusterHandler := clusterconnect.NewClusterServiceHandler(clusterSvc)
+ sc.Mux.Handle(clusterPath, clusterHandler)
+ logger.Infof(ctx, "Mounted ClusterService at %s", clusterPath)
+
+ sc.AddReadyCheck(func(r *http.Request) error {
+ baseContainer := sc.DataStore.GetBaseContainerFQN(r.Context())
+ if baseContainer == "" {
+ return fmt.Errorf("storage connection error")
+ }
+ return nil
+ })
+
+ return nil
+}
diff --git a/dataproxy/test/scripts/create_upload_location.sh b/dataproxy/test/scripts/create_upload_location.sh
new file mode 100755
index 00000000000..27689d32460
--- /dev/null
+++ b/dataproxy/test/scripts/create_upload_location.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+ENDPOINT="${ENDPOINT:-http://localhost:8088}"
+PROJECT="${PROJECT:-testproject}"
+DOMAIN="${DOMAIN:-development}"
+ORG="${ORG:-testorg}"
+FILENAME="${FILENAME:-test-file.txt}"
+FILENAME_ROOT="${FILENAME_ROOT:-test-upload}"
+
+# Create a simple MD5 hash (16 bytes in base64)
+# "test-content-hash" -> base64 encoded
+CONTENT_MD5="dGVzdC1jb250ZW50LWhhc2g="
+
+buf curl --schema . $ENDPOINT/flyteidl2.dataproxy.DataProxyService/CreateUploadLocation --data @- < "/cache/${CACHE_FILE}"; \
+ rm -f /tmp/pg.jar; \
+ # Collect glibc libraries needed by PostgreSQL binaries (not included in zonkyio bundle)
+ mkdir -p /tmp/pg-tmp && tar xJf "/cache/${CACHE_FILE}" -C /tmp/pg-tmp/; \
+ for bin in /tmp/pg-tmp/bin/*; do \
+ ldd "$bin" 2>/dev/null | grep "=>" | awk '{print $3}' | while read lib; do \
+ [ -f "$lib" ] && cp -n "$lib" /glibc-libs/ 2>/dev/null || true; \
+ done; \
+ done; \
+ # Copy dynamic linker
+ cp /lib/ld-linux-aarch64.so.1 /glibc-libs/ 2>/dev/null || true; \
+ cp /lib/aarch64-linux-gnu/ld-linux-aarch64.so.1 /glibc-libs/ 2>/dev/null || true; \
+ cp /lib64/ld-linux-x86-64.so.2 /glibc-libs/ 2>/dev/null || true; \
+ cp /lib/x86_64-linux-gnu/ld-linux-x86-64.so.2 /glibc-libs/ 2>/dev/null || true; \
+ rm -rf /tmp/pg-tmp
+
+
+FROM rancher/k3s:v1.34.6-k3s1
+
+ARG TARGETARCH
+
+ARG FLYTE_DEVBOX_VERSION
+ENV FLYTE_DEVBOX_VERSION "${FLYTE_DEVBOX_VERSION}"
+
+COPY --from=builder /build/images/ /var/lib/rancher/k3s/agent/images/
+COPY images/tar/${TARGETARCH}/ /var/lib/rancher/k3s/agent/images/
+COPY manifests/ /var/lib/rancher/k3s/server/manifests-staging/
+COPY bin/ /bin/
+
+# Install bootstrap and embedded postgres
+COPY --from=bootstrap /flyteorg/build/dist/flyte-devbox-bootstrap /bin/
+COPY --from=bootstrap /flyteorg/build/dist/embedded-postgres /bin/
+
+# Install pre-cached PostgreSQL binaries and glibc libraries
+COPY --from=pg-cache /cache/ /var/cache/embedded-postgres/
+COPY --from=pg-cache /glibc-libs/ /usr/lib/pg-glibc/
+
+# Create dynamic linker symlinks and add postgres user/group
+RUN for f in /usr/lib/pg-glibc/ld-linux-aarch64*; do \
+ [ -f "$f" ] && ln -sf "$f" /lib/$(basename "$f"); \
+ done 2>/dev/null; \
+ for f in /usr/lib/pg-glibc/ld-linux-x86-64*; do \
+ [ -f "$f" ] && mkdir -p /lib64 && ln -sf "$f" /lib64/$(basename "$f"); \
+ done 2>/dev/null; \
+ echo "postgres:x:999:999:PostgreSQL:/tmp:/bin/sh" >> /etc/passwd && \
+ echo "postgres:x:999:" >> /etc/group
+
+ENV LD_LIBRARY_PATH="/usr/lib/pg-glibc"
+
+VOLUME /var/lib/flyte/storage
+
+# Set environment variable for picking up additional CA certificates
+ENV SSL_CERT_DIR /var/lib/flyte/config/ca-certificates
+
+ENTRYPOINT [ "/bin/k3d-entrypoint.sh" ]
+CMD [ "server", "--disable=servicelb", "--disable=metrics-server" ]
diff --git a/docker/devbox-bundled/Dockerfile.gpu b/docker/devbox-bundled/Dockerfile.gpu
new file mode 100644
index 00000000000..346580c9cca
--- /dev/null
+++ b/docker/devbox-bundled/Dockerfile.gpu
@@ -0,0 +1,82 @@
+# syntax=docker/dockerfile:1.7-labs
+#
+# GPU-capable debox cluster image. Layers NVIDIA Container Toolkit + the
+# k8s device-plugin on top of the CPU debox image so everything that ships
+# in the base (flyte-binary, embedded postgres, auto-apply manifests) is
+# inherited verbatim. CI builds the CPU image first and passes its tag in
+# via BASE_IMAGE.
+
+ARG BASE_IMAGE=ghcr.io/flyteorg/flyte-devbox:nightly
+
+
+# Stage NVIDIA Container Toolkit binaries + supporting libs + ldconfig.
+# k3s auto-registers a `nvidia` containerd runtime at startup if
+# /usr/bin/nvidia-container-runtime is on PATH in the final image.
+FROM debian:bookworm-slim AS nvidia-toolkit
+
+ARG TARGETARCH
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ curl gnupg ca-certificates && \
+ rm -rf /var/lib/apt/lists/*
+
+RUN curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
+ | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg && \
+ curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
+ | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
+ > /etc/apt/sources.list.d/nvidia-container-toolkit.list && \
+ apt-get update && \
+ apt-get install -y --no-install-recommends \
+ nvidia-container-toolkit-base \
+ libnvidia-container1 \
+ libnvidia-container-tools && \
+ rm -rf /var/lib/apt/lists/*
+
+# Collect binaries, their shared-lib deps, and the dynamic linker so they run
+# inside the minimal rancher/k3s base (no libc of its own). Also stage
+# /sbin/ldconfig — the toolkit's update-ldcache OCI hook bind-mounts it
+# into workload pods.
+RUN set -ex; \
+ mkdir -p /nvidia-staging/bin /nvidia-staging/lib /nvidia-staging/sbin; \
+ for bin in nvidia-ctk nvidia-container-runtime nvidia-container-runtime.cdi \
+ nvidia-container-runtime.legacy nvidia-container-cli; do \
+ [ -f "/usr/bin/$bin" ] && cp -a "/usr/bin/$bin" /nvidia-staging/bin/ || true; \
+ done; \
+ for bin in /nvidia-staging/bin/*; do \
+ ldd "$bin" 2>/dev/null | grep "=>" | awk '{print $3}' | while read lib; do \
+ [ -f "$lib" ] && cp -n "$lib" /nvidia-staging/lib/ 2>/dev/null || true; \
+ done; \
+ done; \
+ cp /lib64/ld-linux-x86-64.so.2 /nvidia-staging/lib/ 2>/dev/null || true; \
+ cp /lib/ld-linux-aarch64.so.1 /nvidia-staging/lib/ 2>/dev/null || true; \
+ cp /sbin/ldconfig /nvidia-staging/sbin/ldconfig
+
+
+FROM ${BASE_IMAGE}
+
+# Install NVIDIA Container Toolkit binaries + supporting libs. The libs go
+# into a default linker search path (/usr/lib//) because the
+# nvidia-ctk OCI hook is invoked by containerd without inheriting
+# LD_LIBRARY_PATH.
+COPY --from=nvidia-toolkit /nvidia-staging/bin/ /usr/bin/
+COPY --from=nvidia-toolkit /nvidia-staging/lib/ /usr/lib/nvidia/
+COPY --from=nvidia-toolkit /nvidia-staging/sbin/ldconfig /sbin/ldconfig
+RUN ARCH_TRIPLE=$([ "$(uname -m)" = "aarch64" ] && echo "aarch64-linux-gnu" || echo "x86_64-linux-gnu") && \
+ mkdir -p "/usr/lib/${ARCH_TRIPLE}" && \
+ cp -a /usr/lib/nvidia/*.so* "/usr/lib/${ARCH_TRIPLE}/" 2>/dev/null || true
+
+# NVIDIA device-plugin DaemonSet + RuntimeClass (auto-applied by k3s at startup).
+COPY nvidia-device-plugin.yaml /var/lib/rancher/k3s/server/manifests/nvidia-device-plugin.yaml
+
+# k3s reads this template at startup to generate containerd's config.
+# Sets nvidia as the default runtime so GPU pods don't need runtimeClassName.
+COPY containerd-config.toml.tmpl /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl
+
+# Append nvidia libs to the base image's LD_LIBRARY_PATH (which already
+# includes /usr/lib/pg-glibc for embedded postgres).
+ENV LD_LIBRARY_PATH="/usr/lib/pg-glibc:/usr/lib/nvidia"
+
+# Propagate host GPUs into containers scheduled on this node. These env vars
+# are consumed by nvidia-container-runtime.
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
diff --git a/docker/devbox-bundled/Makefile b/docker/devbox-bundled/Makefile
new file mode 100644
index 00000000000..79762823e69
--- /dev/null
+++ b/docker/devbox-bundled/Makefile
@@ -0,0 +1,119 @@
+define FLYTE_BINARY_BUILD
+mkdir -p images/tar/$(1)
+
+docker buildx build \
+ --build-arg FLYTECONSOLE_VERSION=$(FLYTECONSOLE_VERSION) \
+ --builder flyte-devbox \
+ --platform linux/$(1) \
+ --tag flyte-binary-v2:sandbox \
+ --output type=docker,dest=images/tar/$(1)/flyte-binary.tar \
+ ../..
+
+endef
+
+.PHONY: create_builder
+create_builder:
+ [ -n "$(shell docker buildx ls | awk '/^flyte-devbox / {print $$1}')" ] || \
+ docker buildx create --name flyte-devbox \
+ --driver docker-container --driver-opt image=moby/buildkit:master \
+ --buildkitd-flags '--allow-insecure-entitlement security.insecure' \
+ --platform linux/arm64,linux/amd64
+
+.PHONY: flyte
+flyte: FLYTECONSOLE_VERSION := latest
+flyte: create_builder
+ $(foreach arch,amd64 arm64,$(call FLYTE_BINARY_BUILD,$(arch)))
+
+.PHONY: helm-repos
+helm-repos:
+ helm repo add docker-registry https://twuni.github.io/docker-registry.helm
+ helm repo add bitnami https://charts.bitnami.com/bitnami
+ helm repo update
+
+.PHONY: dep_build
+dep_build: helm-repos
+ cd $(SANDBOX_CHART_DIR) && helm dependency build
+
+.PHONY: dep_update
+dep_update: SANDBOX_CHART_DIR := ../../charts/flyte-devbox
+dep_update: dep_build
+ cd $(SANDBOX_CHART_DIR)/charts && for f in *.tgz; do tar xzf "$$f"; done
+
+.PHONY: manifests
+manifests: dep_update
+ mkdir -p manifests
+ kustomize build \
+ --enable-helm \
+ --load-restrictor=LoadRestrictionsNone \
+ kustomize/complete > manifests/complete.yaml
+ kustomize build \
+ --enable-helm \
+ --load-restrictor=LoadRestrictionsNone \
+ kustomize/dev > manifests/dev.yaml
+
+.PHONY: sync-crds
+sync-crds:
+ $(MAKE) -C ../../executor manifests
+
+.PHONY: build
+build: sync-crds flyte dep_update manifests
+ docker buildx build --builder flyte-devbox --allow security.insecure --load \
+ --tag flyte-devbox:latest .
+
+.PHONY: build-gpu
+build-gpu: build
+ docker buildx build --builder flyte-devbox --allow security.insecure --load \
+ --file Dockerfile.gpu \
+ --build-context base=docker-image://flyte-devbox:latest \
+ --build-arg BASE_IMAGE=base \
+ --tag flyte-devbox:gpu-latest .
+
+# Port map
+# 6443 - k8s API server
+# 30000 - Docker Registry
+# 30001 - DB
+# 30002 - RustFS
+# 30080 - Flyte Proxy
+.PHONY: start
+start: FLYTE_DEVBOX_IMAGE := flyte-devbox:latest
+start: FLYTE_DEV := False
+start:
+ [ -n "$(shell docker volume ls --filter name=^flyte-devbox$$ --format {{.Name}})" ] || \
+ docker volume create flyte-devbox
+ @if [ -z "$(shell docker ps --filter name=^flyte-devbox$$ --format {{.Names}})" ]; then \
+ rm -f $(PWD)/.kube/kubeconfig; \
+ docker run --detach --rm --privileged --name flyte-devbox \
+ --add-host "host.docker.internal:host-gateway" \
+ --env FLYTE_DEV=$(FLYTE_DEV) \
+ --env K3S_KUBECONFIG_OUTPUT=/.kube/kubeconfig \
+ --volume $(PWD)/.kube:/.kube \
+ --volume flyte-devbox:/var/lib/flyte/storage \
+ --publish "6443":"6443" \
+ --publish "30000:30000" \
+ --publish "30001:5432" \
+ --publish "30002:30002" \
+ --publish "30080:30080" \
+ $(FLYTE_DEVBOX_IMAGE); \
+ fi
+ @echo "Waiting for kubeconfig..."
+ @until [ -s $(PWD)/.kube/kubeconfig ]; do sleep 1; done
+ @# On WSL, the bind-mounted kubeconfig may be root-owned, which makes the host-side cp fail with permission denied.
+ @docker exec flyte-devbox chown $(shell id -u):$(shell id -g) /.kube/kubeconfig
+ @mkdir -p $(HOME)/.kube
+ @if [ -f $(HOME)/.kube/config ]; then \
+ KUBECONFIG=$(PWD)/.kube/kubeconfig:$(HOME)/.kube/config kubectl config view --flatten > /tmp/kubeconfig-merged && \
+ mv /tmp/kubeconfig-merged $(HOME)/.kube/config; \
+ else \
+ cp $(PWD)/.kube/kubeconfig $(HOME)/.kube/config; \
+ fi
+ @kubectl config use-context flyte-devbox >/dev/null 2>&1 || true
+ @echo "Kubeconfig merged into ~/.kube/config"
+.PHONY: kubeconfig
+.SILENT: kubeconfig
+kubeconfig:
+ sed -i -e "/server:/ s/: .*/: https:\/\/127.0.0.1:$(shell docker port flyte-devbox | grep ^6443 | awk '{print $$3}' | awk -F: '{print $$2}')/" .kube/kubeconfig
+ echo "export KUBECONFIG=$(PWD)/.kube/kubeconfig"
+
+.PHONY: stop
+stop:
+ docker stop --time 5 flyte-devbox
diff --git a/docker/devbox-bundled/bin/k3d-entrypoint-cgroupv2.sh b/docker/devbox-bundled/bin/k3d-entrypoint-cgroupv2.sh
new file mode 100755
index 00000000000..f60369046a9
--- /dev/null
+++ b/docker/devbox-bundled/bin/k3d-entrypoint-cgroupv2.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+#########################################################################################################################################
+# DISCLAIMER #
+# Copied from https://github.com/moby/moby/blob/ed89041433a031cafc0a0f19cfe573c31688d377/hack/dind#L28-L37 #
+# Permission granted by Akihiro Suda (https://github.com/k3d-io/k3d/issues/493#issuecomment-827405962) #
+# Moby License Apache 2.0: https://github.com/moby/moby/blob/ed89041433a031cafc0a0f19cfe573c31688d377/LICENSE #
+#########################################################################################################################################
+if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+ echo "[$(date -Iseconds)] [CgroupV2 Fix] Evacuating Root Cgroup ..."
+ # move the processes from the root group to the /init group,
+ # otherwise writing subtree_control fails with EBUSY.
+ mkdir -p /sys/fs/cgroup/init
+ xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
+ # enable controllers
+ sed -e 's/ / +/g' -e 's/^/+/' <"/sys/fs/cgroup/cgroup.controllers" >"/sys/fs/cgroup/cgroup.subtree_control"
+ echo "[$(date -Iseconds)] [CgroupV2 Fix] Done"
+fi
diff --git a/docker/devbox-bundled/bin/k3d-entrypoint-flyte-devbox-bootstrap.sh b/docker/devbox-bundled/bin/k3d-entrypoint-flyte-devbox-bootstrap.sh
new file mode 100755
index 00000000000..770e9c9ebe8
--- /dev/null
+++ b/docker/devbox-bundled/bin/k3d-entrypoint-flyte-devbox-bootstrap.sh
@@ -0,0 +1,32 @@
+#!/bin/sh
+
+# Fix ownership of PostgreSQL data directory if it exists from a previous run
+# (e.g., old bitnami PostgreSQL used uid 1001, embedded-postgres uses uid 999).
+if [ -d /var/lib/flyte/storage/db ]; then
+ chown -R 999:999 /var/lib/flyte/storage/db
+fi
+
+# Start embedded PostgreSQL in the background (must be running before k3s
+# deploys the flyte-binary pod, which has a wait-for-db init container).
+embedded-postgres &
+
+# Wait for PostgreSQL to be ready before proceeding
+while ! [ -f /tmp/embedded-postgres-ready ]; do
+ sleep 0.5
+done
+
+flyte-devbox-bootstrap
+
+# Wait for K3s to write kubeconfig to the staging path, rename the default
+# context, add a flyte-devbox alias, then copy to the host-mounted output.
+STAGING="${K3S_KUBECONFIG_OUTPUT:-/etc/rancher/k3s/k3s.yaml}"
+(
+ while ! [ -s "$STAGING" ]; do sleep 0.5; done
+ # TODO: Remove flytev2-sandbox after all users have upgraded flyte-sdk.
+ sed -i 's/: default/: flytev2-sandbox/g' "$STAGING"
+ KUBECONFIG="$STAGING" kubectl config set-context flyte-devbox \
+ --cluster=flytev2-sandbox --user=flytev2-sandbox 2>/dev/null || true
+ if [ -n "${KUBECONFIG_FINAL:-}" ] && [ "$KUBECONFIG_FINAL" != "$STAGING" ]; then
+ cp "$STAGING" "$KUBECONFIG_FINAL"
+ fi
+) &
diff --git a/docker/devbox-bundled/bin/k3d-entrypoint.sh b/docker/devbox-bundled/bin/k3d-entrypoint.sh
new file mode 100755
index 00000000000..3044001b52f
--- /dev/null
+++ b/docker/devbox-bundled/bin/k3d-entrypoint.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+LOGFILE="/var/log/k3d-entrypoints_$(date "+%y%m%d%H%M%S").log"
+
+touch "$LOGFILE"
+
+echo "[$(date -Iseconds)] Running k3d entrypoints..." >> "$LOGFILE"
+
+# Redirect K3S_KUBECONFIG_OUTPUT to a staging path so K3s doesn't write
+# directly to the host-mounted path. The bootstrap entrypoint script will
+# post-process the staging file (rename context, add aliases) then copy
+# to the real output path — so the host only sees the finished version.
+if [ -n "${K3S_KUBECONFIG_OUTPUT:-}" ]; then
+ KUBECONFIG_FINAL="$K3S_KUBECONFIG_OUTPUT"
+ export KUBECONFIG_FINAL
+ export K3S_KUBECONFIG_OUTPUT="/tmp/k3s-kubeconfig-staging"
+fi
+
+for entrypoint in /bin/k3d-entrypoint-*.sh ; do
+ echo "[$(date -Iseconds)] Running $entrypoint" >> "$LOGFILE"
+ "$entrypoint" >> "$LOGFILE" 2>&1 || exit 1
+done
+
+echo "[$(date -Iseconds)] Finished k3d entrypoint scripts!" >> "$LOGFILE"
+
+exec /bin/k3s "$@"
diff --git a/docker/devbox-bundled/bootstrap/Makefile b/docker/devbox-bundled/bootstrap/Makefile
new file mode 100644
index 00000000000..05669dffb55
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/Makefile
@@ -0,0 +1,17 @@
+.PHONY: fmt
+fmt:
+ go mod tidy
+ gofmt -w -s .
+ golines -w .
+
+.PHONY: check-fmt
+check-fmt:
+ @[ -z "$(shell gofmt -l .)" ] || ( echo "Not formatted:" && gofmt -l . && exit 1 )
+
+.PHONY: lint
+lint:
+ golangci-lint run
+
+.PHONY: test
+test:
+ go test -v ./...
diff --git a/docker/devbox-bundled/bootstrap/cmd/bootstrap/main.go b/docker/devbox-bundled/bootstrap/cmd/bootstrap/main.go
new file mode 100644
index 00000000000..b7800db680e
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/cmd/bootstrap/main.go
@@ -0,0 +1,113 @@
+package main
+
+import (
+ "fmt"
+ "log"
+ "net"
+ "os"
+ "path/filepath"
+
+ "github.com/flyteorg/flyte/docker/devbox-bundled/bootstrap/internal/transform"
+ "github.com/flyteorg/flyte/docker/devbox-bundled/bootstrap/internal/transform/plugins/config"
+ "github.com/flyteorg/flyte/docker/devbox-bundled/bootstrap/internal/transform/plugins/vars"
+)
+
+const (
+ configDirPath = "/var/lib/flyte/config"
+ configurationConfigMapName = "flyte-devbox-extra-config"
+ deploymentName = "flyte-binary"
+ devModeEnvVar = "FLYTE_DEV"
+ dockerHost = "host.docker.internal"
+ namespace = "flyte"
+
+ // Template paths
+ devTemplatePath = "/var/lib/rancher/k3s/server/manifests-staging/dev.yaml"
+ fullTemplatePath = "/var/lib/rancher/k3s/server/manifests-staging/complete.yaml"
+ renderedManifestPath = "/var/lib/rancher/k3s/server/manifests/flyte.yaml"
+)
+
+// getNodeIP returns the preferred outbound IP address of the node.
+// This is used to create Kubernetes Endpoints that point back to
+// services running on the host (e.g., embedded PostgreSQL).
+func getNodeIP() (string, error) {
+ // Use a UDP dial to determine the preferred outbound IP.
+ // No actual connection is made.
+ conn, err := net.Dial("udp", "8.8.8.8:80")
+ if err != nil {
+ // Fallback: scan interfaces for a non-loopback address
+ return getFirstNonLoopbackIP()
+ }
+ defer conn.Close()
+ localAddr := conn.LocalAddr().(*net.UDPAddr)
+ return localAddr.IP.String(), nil
+}
+
+func getFirstNonLoopbackIP() (string, error) {
+ addrs, err := net.InterfaceAddrs()
+ if err != nil {
+ return "", err
+ }
+ for _, addr := range addrs {
+ if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() && ipnet.IP.To4() != nil {
+ return ipnet.IP.String(), nil
+ }
+ }
+ return "", fmt.Errorf("no suitable IP address found")
+}
+
+func main() {
+ var tmplPath string
+ var tPlugins []transform.Plugin
+
+ if os.Getenv(devModeEnvVar) == "True" {
+ tmplPath = devTemplatePath
+ } else {
+ // If we are not running in dev mode, look for user-specified configuration
+ // to load into the sandbox deployment
+ tmplPath = fullTemplatePath
+
+ cOpts := config.LoaderOpts{
+ ConfigurationConfigMapName: configurationConfigMapName,
+ DeploymentName: deploymentName,
+ Namespace: namespace,
+ DirPath: configDirPath,
+ }
+ c, err := config.NewLoader(&cOpts)
+ if err != nil {
+ log.Fatalf("failed to initialize config loader: %s", err)
+ }
+ tPlugins = append(tPlugins, c)
+ }
+
+ // Replace template variables
+ v := vars.NewVars(map[string]vars.ValueGetter{
+ "%{HOST_GATEWAY_IP}%": func() (string, error) {
+ addrs, err := net.LookupHost(dockerHost)
+ if err != nil {
+ return "", err
+ }
+ return addrs[0], nil
+ },
+ "%{NODE_IP}%": func() (string, error) {
+ return getNodeIP()
+ },
+ })
+ tPlugins = append(tPlugins, v)
+
+ // Render final manifest and write out
+ tmpl, err := os.ReadFile(tmplPath)
+ if err != nil {
+ log.Fatalf("failed to read manifest: %s", err)
+ }
+ t := transform.NewTransformer(tPlugins...)
+ rendered, err := t.Transform(tmpl)
+ if err != nil {
+ log.Fatalf("failed to apply transformations: %s", err)
+ }
+ if err := os.MkdirAll(filepath.Dir(renderedManifestPath), 0755); err != nil {
+ log.Fatalf("failed to create destination directory: %s", err)
+ }
+ if err := os.WriteFile(renderedManifestPath, rendered, 0644); err != nil {
+ log.Fatalf("failed to write rendered manifest: %s", err)
+ }
+}
diff --git a/docker/devbox-bundled/bootstrap/cmd/bootstrap/main_test.go b/docker/devbox-bundled/bootstrap/cmd/bootstrap/main_test.go
new file mode 100644
index 00000000000..36147c4b1f3
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/cmd/bootstrap/main_test.go
@@ -0,0 +1,33 @@
+package main
+
+import (
+ "net"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetNodeIP(t *testing.T) {
+ ip, err := getNodeIP()
+ require.NoError(t, err)
+ assert.NotEmpty(t, ip)
+
+ // Should be a valid IPv4 address
+ parsed := net.ParseIP(ip)
+ assert.NotNil(t, parsed, "should be a valid IP address")
+ assert.NotNil(t, parsed.To4(), "should be an IPv4 address")
+
+ // Should not be loopback
+ assert.False(t, parsed.IsLoopback(), "should not be a loopback address")
+}
+
+func TestGetFirstNonLoopbackIP(t *testing.T) {
+ ip, err := getFirstNonLoopbackIP()
+ require.NoError(t, err)
+ assert.NotEmpty(t, ip)
+
+ parsed := net.ParseIP(ip)
+ assert.NotNil(t, parsed, "should be a valid IP address")
+ assert.False(t, parsed.IsLoopback(), "should not be a loopback address")
+}
diff --git a/docker/devbox-bundled/bootstrap/cmd/embedded-postgres/main.go b/docker/devbox-bundled/bootstrap/cmd/embedded-postgres/main.go
new file mode 100644
index 00000000000..def043c4967
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/cmd/embedded-postgres/main.go
@@ -0,0 +1,160 @@
+package main
+
+import (
+ "database/sql"
+ "fmt"
+ "log"
+ "os"
+ "os/signal"
+ "path/filepath"
+ "syscall"
+
+ embeddedpostgres "github.com/fergusstrange/embedded-postgres"
+ _ "github.com/lib/pq"
+)
+
+const (
+ defaultPort = 5432
+ defaultUser = "postgres"
+ defaultPass = "postgres"
+ cachePath = "/var/cache/embedded-postgres"
+ runtimePath = "/tmp/embedded-postgres-runtime"
+ dataPath = "/var/lib/flyte/storage/db"
+ pgUID = 999
+ pgGID = 999
+)
+
+func main() {
+ log.Println("Starting embedded PostgreSQL...")
+
+ // Prepare directories as root before dropping privileges.
+ // The library calls os.RemoveAll(dataPath) on start, which requires
+ // write permission on the parent directory, so we chown parents too.
+ for _, dir := range []string{dataPath, runtimePath, cachePath} {
+ if err := os.MkdirAll(dir, 0700); err != nil {
+ log.Fatalf("Failed to create directory %s: %v", dir, err)
+ }
+ // Chown both the directory and its parent (needed for RemoveAll)
+ for _, d := range []string{dir, filepath.Dir(dir)} {
+ if err := os.Chown(d, pgUID, pgGID); err != nil {
+ log.Fatalf("Failed to chown directory %s: %v", d, err)
+ }
+ }
+ }
+
+ // Drop privileges to postgres user (PostgreSQL refuses to run as root)
+ if os.Getuid() == 0 {
+ if err := syscall.Setgid(pgGID); err != nil {
+ log.Fatalf("Failed to setgid: %v", err)
+ }
+ if err := syscall.Setuid(pgUID); err != nil {
+ log.Fatalf("Failed to setuid: %v", err)
+ }
+ log.Printf("Dropped privileges to uid=%d gid=%d", os.Getuid(), os.Getgid())
+ }
+
+ db := embeddedpostgres.NewDatabase(
+ embeddedpostgres.DefaultConfig().
+ Port(uint32(defaultPort)).
+ Username(defaultUser).
+ Password(defaultPass).
+ Database("postgres").
+ DataPath(dataPath).
+ RuntimePath(runtimePath).
+ CachePath(cachePath).
+ StartParameters(map[string]string{
+ "max_connections": "200",
+ "listen_addresses": "*",
+ }).
+ Version(embeddedpostgres.V16),
+ )
+
+ if err := db.Start(); err != nil {
+ log.Fatalf("Failed to start embedded PostgreSQL: %v", err)
+ }
+ log.Printf("Embedded PostgreSQL started on port %d", defaultPort)
+
+ // Allow connections from all hosts (needed for K8s pods to connect)
+ if err := enableRemoteConnections(); err != nil {
+ log.Fatalf("Failed to configure pg_hba.conf: %v", err)
+ }
+
+ // Create additional databases needed by Flyte
+ if err := createDatabases(); err != nil {
+ log.Fatalf("Failed to create databases: %v", err)
+ }
+
+ // Signal readiness
+ if err := os.WriteFile("/tmp/embedded-postgres-ready", []byte("ready"), 0644); err != nil {
+ log.Printf("Warning: failed to write ready file: %v", err)
+ }
+ log.Println("Embedded PostgreSQL is ready")
+
+ // Wait for shutdown signal
+ sigCh := make(chan os.Signal, 1)
+ signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+ sig := <-sigCh
+
+ log.Printf("Received signal %v, shutting down embedded PostgreSQL...", sig)
+ if err := db.Stop(); err != nil {
+ log.Printf("Warning: failed to stop embedded PostgreSQL cleanly: %v", err)
+ }
+ log.Println("Embedded PostgreSQL stopped")
+}
+
+func enableRemoteConnections() error {
+ hbaPath := filepath.Join(dataPath, "pg_hba.conf")
+ f, err := os.OpenFile(hbaPath, os.O_APPEND|os.O_WRONLY, 0600)
+ if err != nil {
+ return fmt.Errorf("failed to open pg_hba.conf: %w", err)
+ }
+ defer f.Close()
+ if _, err := f.WriteString("\n# Allow all hosts (sandbox environment)\nhost all all 0.0.0.0/0 password\nhost all all ::/0 password\n"); err != nil {
+ return fmt.Errorf("failed to write pg_hba.conf: %w", err)
+ }
+
+ // Reload PostgreSQL configuration
+ connStr := fmt.Sprintf(
+ "host=127.0.0.1 port=%d user=%s password=%s dbname=postgres sslmode=disable",
+ defaultPort, defaultUser, defaultPass,
+ )
+ conn, err := sql.Open("postgres", connStr)
+ if err != nil {
+ return fmt.Errorf("failed to connect for reload: %w", err)
+ }
+ defer conn.Close()
+ if _, err := conn.Exec("SELECT pg_reload_conf()"); err != nil {
+ return fmt.Errorf("failed to reload config: %w", err)
+ }
+ log.Println("Configured pg_hba.conf for remote connections")
+ return nil
+}
+
+func createDatabases() error {
+ connStr := fmt.Sprintf(
+ "host=127.0.0.1 port=%d user=%s password=%s dbname=postgres sslmode=disable",
+ defaultPort, defaultUser, defaultPass,
+ )
+ conn, err := sql.Open("postgres", connStr)
+ if err != nil {
+ return fmt.Errorf("failed to connect to postgres: %w", err)
+ }
+ defer conn.Close()
+
+ for _, dbName := range []string{"flyte", "runs"} {
+ var exists bool
+ err := conn.QueryRow("SELECT EXISTS(SELECT 1 FROM pg_database WHERE datname = $1)", dbName).Scan(&exists)
+ if err != nil {
+ return fmt.Errorf("failed to check database %s: %w", dbName, err)
+ }
+ if !exists {
+ if _, err := conn.Exec(fmt.Sprintf("CREATE DATABASE %s", dbName)); err != nil {
+ return fmt.Errorf("failed to create database %s: %w", dbName, err)
+ }
+ log.Printf("Created database: %s", dbName)
+ } else {
+ log.Printf("Database already exists: %s", dbName)
+ }
+ }
+ return nil
+}
diff --git a/docker/devbox-bundled/bootstrap/cmd/embedded-postgres/main_test.go b/docker/devbox-bundled/bootstrap/cmd/embedded-postgres/main_test.go
new file mode 100644
index 00000000000..efee4c938e5
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/cmd/embedded-postgres/main_test.go
@@ -0,0 +1,63 @@
+package main
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestConstants(t *testing.T) {
+ assert.Equal(t, 5432, defaultPort)
+ assert.Equal(t, "postgres", defaultUser)
+ assert.Equal(t, "postgres", defaultPass)
+ assert.Equal(t, "/var/cache/embedded-postgres", cachePath)
+ assert.Equal(t, "/tmp/embedded-postgres-runtime", runtimePath)
+ assert.Equal(t, "/var/lib/flyte/storage/db", dataPath)
+ assert.Equal(t, 999, pgUID)
+ assert.Equal(t, 999, pgGID)
+}
+
+func TestDataPathPersistence(t *testing.T) {
+ // Verify the data path would survive container restarts (under /var/lib/flyte/storage/)
+ assert.Contains(t, dataPath, "/var/lib/flyte/storage/")
+}
+
+func TestInitDBSkipsIfVersionFileExists(t *testing.T) {
+ // Create a temp directory with PG_VERSION file to simulate existing data dir
+ tmpDir := t.TempDir()
+ dataDir := filepath.Join(tmpDir, "db")
+ err := os.MkdirAll(dataDir, 0700)
+ assert.NoError(t, err)
+
+ versionFile := filepath.Join(dataDir, "PG_VERSION")
+ err = os.WriteFile(versionFile, []byte("16"), 0644)
+ assert.NoError(t, err)
+
+ // The library handles this internally - just verify the file check works
+ _, err = os.Stat(versionFile)
+ assert.NoError(t, err, "PG_VERSION file should exist")
+}
+
+func TestPgHbaConfAppend(t *testing.T) {
+ // Simulate pg_hba.conf modification that enableRemoteConnections does
+ tmpDir := t.TempDir()
+ hbaPath := filepath.Join(tmpDir, "pg_hba.conf")
+
+ initialContent := "# default pg_hba.conf\nlocal all all password\n"
+ err := os.WriteFile(hbaPath, []byte(initialContent), 0600)
+ assert.NoError(t, err)
+
+ f, err := os.OpenFile(hbaPath, os.O_APPEND|os.O_WRONLY, 0600)
+ assert.NoError(t, err)
+ _, err = f.WriteString("\n# Allow all hosts (sandbox environment)\nhost all all 0.0.0.0/0 password\nhost all all ::/0 password\n")
+ assert.NoError(t, err)
+ f.Close()
+
+ content, err := os.ReadFile(hbaPath)
+ assert.NoError(t, err)
+ assert.Contains(t, string(content), "0.0.0.0/0")
+ assert.Contains(t, string(content), "::/0")
+ assert.Contains(t, string(content), initialContent) // original content preserved
+}
diff --git a/docker/devbox-bundled/bootstrap/go.mod b/docker/devbox-bundled/bootstrap/go.mod
new file mode 100644
index 00000000000..2d99c2f896c
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/go.mod
@@ -0,0 +1,56 @@
+module github.com/flyteorg/flyte/docker/devbox-bundled/bootstrap
+
+go 1.24.0
+
+require (
+ github.com/fergusstrange/embedded-postgres v1.34.0
+ github.com/lib/pq v1.12.1
+ github.com/stretchr/testify v1.10.0
+ k8s.io/api v0.26.1
+ k8s.io/apimachinery v0.26.1
+ sigs.k8s.io/kustomize/api v0.12.1
+ sigs.k8s.io/kustomize/kyaml v0.13.9
+ sigs.k8s.io/yaml v1.3.0
+)
+
+require (
+ github.com/PuerkitoBio/purell v1.1.1 // indirect
+ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
+ github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+ github.com/go-errors/errors v1.0.1 // indirect
+ github.com/go-logr/logr v1.2.3 // indirect
+ github.com/go-openapi/jsonpointer v0.19.5 // indirect
+ github.com/go-openapi/jsonreference v0.19.5 // indirect
+ github.com/go-openapi/swag v0.19.14 // indirect
+ github.com/gogo/protobuf v1.3.2 // indirect
+ github.com/golang/protobuf v1.5.2 // indirect
+ github.com/google/gnostic v0.5.7-v3refs // indirect
+ github.com/google/go-cmp v0.5.9 // indirect
+ github.com/google/gofuzz v1.1.0 // indirect
+ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+ github.com/imdario/mergo v0.3.6 // indirect
+ github.com/josharian/intern v1.0.0 // indirect
+ github.com/json-iterator/go v1.1.12 // indirect
+ github.com/mailru/easyjson v0.7.6 // indirect
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+ github.com/modern-go/reflect2 v1.0.2 // indirect
+ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
+ github.com/pkg/errors v0.9.1 // indirect
+ github.com/pmezard/go-difflib v1.0.0 // indirect
+ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
+ github.com/xlab/treeprint v1.1.0 // indirect
+ go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
+ golang.org/x/net v0.38.0 // indirect
+ golang.org/x/sys v0.31.0 // indirect
+ golang.org/x/text v0.23.0 // indirect
+ google.golang.org/protobuf v1.33.0 // indirect
+ gopkg.in/inf.v0 v0.9.1 // indirect
+ gopkg.in/yaml.v2 v2.4.0 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
+ k8s.io/klog/v2 v2.80.1 // indirect
+ k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
+ k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect
+ sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+ sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+)
diff --git a/docker/devbox-bundled/bootstrap/go.sum b/docker/devbox-bundled/bootstrap/go.sum
new file mode 100644
index 00000000000..d3e10f5f49d
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/go.sum
@@ -0,0 +1,229 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
+github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fergusstrange/embedded-postgres v1.34.0 h1:c6RKhPKFsLVU+Tdxsx8q0UxCHsvZZ/iShAnljRBXs6s=
+github.com/fergusstrange/embedded-postgres v1.34.0/go.mod h1:w0YvnCgf19o6tskInrOOACtnqfVlOvluz3hlNLY7tRk=
+github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
+github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
+github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
+github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM=
+github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
+github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54=
+github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
+github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
+github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lib/pq v1.12.1 h1:x1nbl/338GLqeDJ/FAiILallhAsqubLzEZu/pXtHUow=
+github.com/lib/pq v1.12.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
+github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
+github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
+github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk=
+github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc=
+go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
+golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
+golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ=
+k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg=
+k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ=
+k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74=
+k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
+k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=
+k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4=
+k8s.io/utils v0.0.0-20221107191617-1a15be271d1d h1:0Smp/HP1OH4Rvhe+4B8nWGERtlqAGSftbSbbmm45oFs=
+k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k=
+sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM=
+sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s=
+sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk=
+sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
+sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
diff --git a/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/cluster_resource_templates.go b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/cluster_resource_templates.go
new file mode 100644
index 00000000000..4731820f2ce
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/cluster_resource_templates.go
@@ -0,0 +1,135 @@
+package config
+
+import (
+ "encoding/hex"
+ "fmt"
+ "os"
+ "path/filepath"
+
+ "github.com/flyteorg/flyte/docker/devbox-bundled/bootstrap/internal/utils"
+ appsv1 "k8s.io/api/apps/v1"
+ apiv1 "k8s.io/api/core/v1"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "sigs.k8s.io/kustomize/api/types"
+)
+
+type ClusterResourceTemplatesNotFound struct {
+ path string
+}
+
+func (e *ClusterResourceTemplatesNotFound) Error() string {
+ return fmt.Sprintf("cluster resource templates directory not found or is empty: %s", e.path)
+}
+
+type ClusterResourceTemplates struct {
+ ConfigMapName string
+ DeploymentName string
+ Namespace string
+ Paths []string
+}
+
+func NewClusterResourceTemplates(
+ configMapName, deploymentName, namespace, dirPath string,
+) (*ClusterResourceTemplates, error) {
+ entries, err := os.ReadDir(dirPath)
+ if os.IsNotExist(err) {
+ return nil, &ClusterResourceTemplatesNotFound{dirPath}
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ var paths []string
+ for _, entry := range entries {
+ if entry.IsDir() {
+ continue
+ }
+ info, err := entry.Info()
+ if err != nil {
+ return nil, err
+ }
+ if info.Size() > 0 {
+ paths = append(paths, filepath.Join(dirPath, entry.Name()))
+ }
+ }
+
+ if len(paths) == 0 {
+ return nil, &ClusterResourceTemplatesNotFound{dirPath}
+ }
+
+ return &ClusterResourceTemplates{
+ ConfigMapName: configMapName,
+ DeploymentName: deploymentName,
+ Namespace: namespace,
+ Paths: paths,
+ }, nil
+}
+
+func (c *ClusterResourceTemplates) Update(k *types.Kustomization) error {
+ // Build ConfigMap data from files
+ data := map[string]string{}
+ for _, path := range c.Paths {
+ content, err := os.ReadFile(path)
+ if err != nil {
+ return fmt.Errorf("reading cluster resource template %s: %w", path, err)
+ }
+ data[filepath.Base(path)] = string(content)
+ }
+
+ // Patch the ConfigMap directly with file contents
+ cm := corev1.ConfigMap{
+ TypeMeta: metav1.TypeMeta{
+ APIVersion: "v1",
+ Kind: "ConfigMap",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: c.ConfigMapName,
+ Namespace: c.Namespace,
+ },
+ Data: data,
+ }
+ cmPatchYaml, err := utils.MarshalPatch(&cm, true, true)
+ if err != nil {
+ return err
+ }
+
+ // Patch the Deployment with a checksum annotation
+ checksum, err := utils.FileCollectionChecksum(c.Paths)
+ if err != nil {
+ return err
+ }
+ deployPatch := appsv1.Deployment{
+ TypeMeta: metav1.TypeMeta{
+ Kind: "Deployment",
+ APIVersion: "apps/v1",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: c.DeploymentName,
+ Namespace: c.Namespace,
+ },
+ Spec: appsv1.DeploymentSpec{
+ Template: apiv1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ "checksum/extra-cluster-resource-templates": hex.EncodeToString(checksum),
+ },
+ },
+ },
+ },
+ }
+ deployPatchYaml, err := utils.MarshalPatch(&deployPatch, true, true)
+ if err != nil {
+ return err
+ }
+
+ if k.Patches == nil {
+ k.Patches = []types.Patch{}
+ }
+ k.Patches = append(k.Patches,
+ types.Patch{Patch: string(cmPatchYaml)},
+ types.Patch{Patch: string(deployPatchYaml)},
+ )
+
+ return nil
+}
diff --git a/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/configuration.go b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/configuration.go
new file mode 100644
index 00000000000..e32c4ae293f
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/configuration.go
@@ -0,0 +1,108 @@
+package config
+
+import (
+ "encoding/hex"
+ "fmt"
+ "os"
+
+ "github.com/flyteorg/flyte/docker/devbox-bundled/bootstrap/internal/utils"
+ appsv1 "k8s.io/api/apps/v1"
+ apiv1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "sigs.k8s.io/kustomize/api/types"
+)
+
+type ConfigurationNotFound struct {
+ path string
+}
+
+func (e *ConfigurationNotFound) Error() string {
+ return fmt.Sprintf("configuration file not found or is empty: %s", e.path)
+}
+
+type Configuration struct {
+ ConfigMapName string
+ DeploymentName string
+ Namespace string
+ Path string
+}
+
+func NewConfiguration(
+ configMapName, deploymentName, namespace, path string,
+) (*Configuration, error) {
+ // Check that the file is accessible and not empty
+ info, err := os.Stat(path)
+ if os.IsNotExist(err) || info.Size() == 0 {
+ return nil, &ConfigurationNotFound{path}
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ return &Configuration{
+ ConfigMapName: configMapName,
+ DeploymentName: deploymentName,
+ Namespace: namespace,
+ Path: path,
+ }, nil
+}
+
+func (c *Configuration) checksum() ([]byte, error) {
+ checksum, err := utils.FileChecksum(c.Path)
+ if err != nil {
+ return nil, err
+ }
+ return checksum, nil
+}
+
+func (c *Configuration) Update(k *types.Kustomization) error {
+ // Add configmap args
+ if k.ConfigMapGenerator == nil {
+ k.ConfigMapGenerator = []types.ConfigMapArgs{}
+ }
+ k.ConfigMapGenerator = append(k.ConfigMapGenerator, types.ConfigMapArgs{
+ GeneratorArgs: types.GeneratorArgs{
+ Namespace: c.Namespace,
+ Name: c.ConfigMapName,
+ Behavior: "replace",
+ KvPairSources: types.KvPairSources{
+ FileSources: []string{fmt.Sprintf("999-extra-config.yaml=%s", c.Path)},
+ },
+ },
+ })
+
+ // Patch deployment to add annotation
+ checksum, err := c.checksum()
+ if err != nil {
+ return err
+ }
+ patch := appsv1.Deployment{
+ TypeMeta: metav1.TypeMeta{
+ Kind: "Deployment",
+ APIVersion: "apps/v1",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: c.DeploymentName,
+ Namespace: c.Namespace,
+ },
+ Spec: appsv1.DeploymentSpec{
+ Template: apiv1.PodTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: map[string]string{
+ "checksum/extra-configuration": hex.EncodeToString(checksum),
+ },
+ },
+ },
+ },
+ }
+ patchYaml, err := utils.MarshalPatch(&patch, true, true)
+ if err != nil {
+ return err
+ }
+ if k.Patches == nil {
+ k.Patches = []types.Patch{}
+ }
+ k.Patches = append(k.Patches, types.Patch{Patch: string(patchYaml)})
+
+ return nil
+}
diff --git a/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/loader.go b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/loader.go
new file mode 100644
index 00000000000..5b7df680810
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/loader.go
@@ -0,0 +1,112 @@
+package config
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+
+ "sigs.k8s.io/kustomize/api/krusty"
+ "sigs.k8s.io/kustomize/api/types"
+ "sigs.k8s.io/kustomize/kyaml/filesys"
+ "sigs.k8s.io/yaml"
+)
+
+type LoaderOpts struct {
+ ConfigurationConfigMapName string
+ ClusterResourceTemplatesConfigMapName string
+ DeploymentName string
+ Namespace string
+ DirPath string
+}
+
+type Loader struct {
+ configuration *Configuration
+ clusterResourceTemplates *ClusterResourceTemplates
+}
+
+func NewLoader(opts *LoaderOpts) (*Loader, error) {
+ var err error
+ loader := Loader{}
+
+ absDirPath, err := filepath.Abs(opts.DirPath)
+ if err != nil {
+ return nil, err
+ }
+
+ loader.configuration, err = NewConfiguration(
+ opts.ConfigurationConfigMapName,
+ opts.DeploymentName,
+ opts.Namespace,
+ filepath.Join(absDirPath, "config.yaml"),
+ )
+ var configurationNotFound *ConfigurationNotFound
+ if err != nil && !errors.As(err, &configurationNotFound) {
+ return nil, err
+ }
+
+ loader.clusterResourceTemplates, err = NewClusterResourceTemplates(
+ opts.ClusterResourceTemplatesConfigMapName,
+ opts.DeploymentName,
+ opts.Namespace,
+ filepath.Join(absDirPath, "cluster-resource-templates"),
+ )
+ var clusterResourceTemplatesNotFound *ClusterResourceTemplatesNotFound
+ if err != nil && !errors.As(err, &clusterResourceTemplatesNotFound) {
+ return nil, err
+ }
+
+ return &loader, nil
+}
+
+func (cl *Loader) Transform(data []byte) ([]byte, error) {
+ if cl.configuration == nil && cl.clusterResourceTemplates == nil {
+ return data, nil
+ }
+
+ workDir, err := os.MkdirTemp("", "")
+ if err != nil {
+ return nil, err
+ }
+ defer os.RemoveAll(workDir)
+
+ baseManifestPath := filepath.Join(workDir, "base.yaml")
+ if err := os.WriteFile(baseManifestPath, data, 0644); err != nil {
+ return nil, err
+ }
+
+ k := types.Kustomization{Resources: []string{baseManifestPath}}
+
+ if cl.clusterResourceTemplates != nil {
+ if err := cl.clusterResourceTemplates.Update(&k); err != nil {
+ return nil, err
+ }
+ }
+
+ if cl.configuration != nil {
+ if err := cl.configuration.Update(&k); err != nil {
+ return nil, err
+ }
+ }
+
+ kyaml, err := yaml.Marshal(k)
+ if err != nil {
+ return nil, err
+ }
+ if err := os.WriteFile(filepath.Join(workDir, "kustomization.yaml"), kyaml, 0644); err != nil {
+ return nil, err
+ }
+
+ opts := krusty.MakeDefaultOptions()
+ opts.DoLegacyResourceSort = true
+ opts.LoadRestrictions = types.LoadRestrictionsNone
+ kustomizer := krusty.MakeKustomizer(opts)
+ resMap, err := kustomizer.Run(filesys.MakeFsOnDisk(), workDir)
+ if err != nil {
+ return nil, err
+ }
+ out, err := resMap.AsYaml()
+ if err != nil {
+ return nil, err
+ }
+ return out, nil
+}
diff --git a/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/loader_test.go b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/loader_test.go
new file mode 100644
index 00000000000..b337e74b185
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/loader_test.go
@@ -0,0 +1,124 @@
+package config
+
+import (
+ "encoding/hex"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/flyteorg/flyte/docker/devbox-bundled/bootstrap/internal/utils"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestLoaderHappy(t *testing.T) {
+ cOpts := LoaderOpts{
+ ConfigurationConfigMapName: "test-config",
+ ClusterResourceTemplatesConfigMapName: "test-cluster-resource-templates",
+ DeploymentName: "test-deployment",
+ Namespace: "test",
+ DirPath: filepath.Join("testdata", "happy"),
+ }
+ c, err := NewLoader(&cOpts)
+ require.NoError(t, err)
+
+ base, err := os.ReadFile(filepath.Join("testdata", "base.yaml"))
+ require.NoError(t, err)
+
+ rendered, err := c.Transform(base)
+ require.NoError(t, err)
+
+ configAbsPath, err := filepath.Abs(filepath.Join("testdata", "happy", "config.yaml"))
+ require.NoError(t, err)
+ configChecksum, err := utils.FileChecksum(configAbsPath)
+ require.NoError(t, err)
+
+ crtAbsPath, err := filepath.Abs(filepath.Join("testdata", "happy", "cluster-resource-templates", "resource.yaml"))
+ require.NoError(t, err)
+ crtChecksum, err := utils.FileCollectionChecksum([]string{crtAbsPath})
+ require.NoError(t, err)
+
+ expected := "apiVersion: v1\n" +
+ "data:\n" +
+ " resource.yaml: |\n" +
+ " foo: bar\n" +
+ "kind: ConfigMap\n" +
+ "metadata:\n" +
+ " name: test-cluster-resource-templates\n" +
+ " namespace: test\n" +
+ "---\n" +
+ "apiVersion: v1\n" +
+ "data:\n" +
+ " 999-extra-config.yaml: |\n" +
+ " ham: spam\n" +
+ "kind: ConfigMap\n" +
+ "metadata:\n" +
+ " name: test-config\n" +
+ " namespace: test\n" +
+ "---\n" +
+ "apiVersion: apps/v1\n" +
+ "kind: Deployment\n" +
+ "metadata:\n" +
+ " name: test-deployment\n" +
+ " namespace: test\n" +
+ "spec:\n" +
+ " replicas: 1\n" +
+ " selector:\n" +
+ " matchLabels:\n" +
+ " app: test\n" +
+ " template:\n" +
+ " metadata:\n" +
+ " annotations:\n" +
+ " checksum/extra-cluster-resource-templates: " + hex.EncodeToString(crtChecksum) + "\n" +
+ " checksum/extra-configuration: " + hex.EncodeToString(configChecksum) + "\n" +
+ " labels:\n" +
+ " app: test\n" +
+ " spec:\n" +
+ " containers:\n" +
+ " - image: test:test\n" +
+ " name: test\n"
+
+ assert.Equal(t, expected, string(rendered))
+}
+
+func TestLoaderEmptyDir(t *testing.T) {
+ cOpts := LoaderOpts{
+ ConfigurationConfigMapName: "test-config",
+ ClusterResourceTemplatesConfigMapName: "test-cluster-resource-templates",
+ DeploymentName: "test-deployment",
+ Namespace: "test",
+ DirPath: filepath.Join("testdata", "emptydir"),
+ }
+ c, err := NewLoader(&cOpts)
+ require.NoError(t, err)
+ assert.Nil(t, c.configuration)
+ assert.Nil(t, c.clusterResourceTemplates)
+
+ base, err := os.ReadFile(filepath.Join("testdata", "base.yaml"))
+ require.NoError(t, err)
+
+ rendered, err := c.Transform(base)
+ require.NoError(t, err)
+ assert.Equal(t, base, rendered)
+}
+
+func TestLoaderEmptyFiles(t *testing.T) {
+ cOpts := LoaderOpts{
+ ConfigurationConfigMapName: "test-config",
+ ClusterResourceTemplatesConfigMapName: "test-cluster-resource-templates",
+ DeploymentName: "test-deployment",
+ Namespace: "test",
+ DirPath: filepath.Join("testdata", "emptyfile"),
+ }
+ c, err := NewLoader(&cOpts)
+ require.NoError(t, err)
+ assert.Nil(t, c.configuration)
+ assert.Nil(t, c.clusterResourceTemplates)
+
+ base, err := os.ReadFile(filepath.Join("testdata", "base.yaml"))
+ require.NoError(t, err)
+
+ rendered, err := c.Transform(base)
+ require.NoError(t, err)
+ assert.Equal(t, base, rendered)
+}
diff --git a/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/base.yaml b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/base.yaml
new file mode 100644
index 00000000000..566572ec47f
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/base.yaml
@@ -0,0 +1,30 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: test-cluster-resource-templates
+ namespace: test
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: test-config
+ namespace: test
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: test-deployment
+ namespace: test
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: test
+ template:
+ metadata:
+ labels:
+ app: test
+ spec:
+ containers:
+ - name: test
+ image: test:test
diff --git a/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/emptydir/.keep b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/emptydir/.keep
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/emptyfile/cluster-resource-templates/resource.yaml b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/emptyfile/cluster-resource-templates/resource.yaml
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/emptyfile/config.yaml b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/emptyfile/config.yaml
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/happy/cluster-resource-templates/resource.yaml b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/happy/cluster-resource-templates/resource.yaml
new file mode 100644
index 00000000000..20e9ff3feaa
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/happy/cluster-resource-templates/resource.yaml
@@ -0,0 +1 @@
+foo: bar
diff --git a/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/happy/config.yaml b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/happy/config.yaml
new file mode 100644
index 00000000000..874969518b5
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/transform/plugins/config/testdata/happy/config.yaml
@@ -0,0 +1 @@
+ham: spam
diff --git a/docker/devbox-bundled/bootstrap/internal/transform/plugins/vars/vars.go b/docker/devbox-bundled/bootstrap/internal/transform/plugins/vars/vars.go
new file mode 100644
index 00000000000..105d679c340
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/transform/plugins/vars/vars.go
@@ -0,0 +1,28 @@
+package vars
+
+import (
+ "strings"
+)
+
+type ValueGetter func() (string, error)
+
+type Vars struct {
+ internal map[string]ValueGetter
+}
+
+func NewVars(values map[string]ValueGetter) *Vars {
+ return &Vars{internal: values}
+}
+
+func (v *Vars) Transform(data []byte) ([]byte, error) {
+ var tokens []string
+ for key, valueGetter := range v.internal {
+ value, err := valueGetter()
+ if err != nil {
+ return nil, err
+ }
+ tokens = append(tokens, []string{key, value}...)
+ }
+ replacer := strings.NewReplacer(tokens...)
+ return []byte(replacer.Replace(string(data))), nil
+}
diff --git a/docker/devbox-bundled/bootstrap/internal/transform/plugins/vars/vars_test.go b/docker/devbox-bundled/bootstrap/internal/transform/plugins/vars/vars_test.go
new file mode 100644
index 00000000000..fdcfe9329de
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/transform/plugins/vars/vars_test.go
@@ -0,0 +1,23 @@
+package vars
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestVars(t *testing.T) {
+ v := NewVars(map[string]ValueGetter{
+ "%(foo)%": func() (string, error) {
+ return "there", nil
+ },
+ "%(bar)%": func() (string, error) {
+ return "friend", nil
+ },
+ })
+ result, err := v.Transform([]byte("hello %(foo)%, %(bar)%"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(t, result, []byte("hello there, friend"), "strings should match")
+}
diff --git a/docker/devbox-bundled/bootstrap/internal/transform/transformer.go b/docker/devbox-bundled/bootstrap/internal/transform/transformer.go
new file mode 100644
index 00000000000..749297b46b0
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/transform/transformer.go
@@ -0,0 +1,26 @@
+package transform
+
+type Plugin interface {
+ Transform(data []byte) ([]byte, error)
+}
+
+type Transformer struct {
+ plugins []Plugin
+}
+
+func NewTransformer(plugins ...Plugin) *Transformer {
+ return &Transformer{plugins: plugins}
+}
+
+func (t *Transformer) Transform(data []byte) ([]byte, error) {
+ dataCopy := make([]byte, len(data))
+ copy(dataCopy, data)
+
+ var err error
+ for _, p := range t.plugins {
+ if dataCopy, err = p.Transform(dataCopy); err != nil {
+ return nil, err
+ }
+ }
+ return dataCopy, nil
+}
diff --git a/docker/devbox-bundled/bootstrap/internal/transform/transformer_test.go b/docker/devbox-bundled/bootstrap/internal/transform/transformer_test.go
new file mode 100644
index 00000000000..bb8b805a0ef
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/transform/transformer_test.go
@@ -0,0 +1,34 @@
+package transform
+
+import (
+ "bytes"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+type Foo struct{}
+
+func (f Foo) Transform(data []byte) ([]byte, error) {
+ return bytes.Join([][]byte{[]byte("foo transformed:"), data}, []byte(" ")), nil
+}
+
+type Bar struct{}
+
+func (b Bar) Transform(data []byte) ([]byte, error) {
+ return bytes.Join([][]byte{[]byte("bar transformed:"), data}, []byte(" ")), nil
+}
+
+func TestTransformer(t *testing.T) {
+ tx := NewTransformer(Foo{}, Bar{})
+ result, err := tx.Transform([]byte("test"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(
+ t,
+ result,
+ []byte("bar transformed: foo transformed: test"),
+ "strings should match",
+ )
+}
diff --git a/docker/devbox-bundled/bootstrap/internal/utils/checksum.go b/docker/devbox-bundled/bootstrap/internal/utils/checksum.go
new file mode 100644
index 00000000000..d47bd1fd37d
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/utils/checksum.go
@@ -0,0 +1,44 @@
+package utils
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "io"
+ "os"
+ "sort"
+ "strings"
+)
+
+func FileChecksum(path string) ([]byte, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+
+ h := sha256.New()
+ if _, err := io.Copy(h, f); err != nil {
+ return nil, err
+ }
+
+ return h.Sum(nil), nil
+}
+
+func FileCollectionChecksum(paths []string) ([]byte, error) {
+ // Compute individual checksums and sort
+ var checksumPaths []string
+ for _, p := range paths {
+ c, err := FileChecksum(p)
+ if err != nil {
+ return nil, err
+ }
+ checksumPaths = append(checksumPaths, fmt.Sprintf("%s\t%s", hex.EncodeToString(c), p))
+ }
+ sort.Strings(checksumPaths)
+
+ // Compute checksum of sorted checksum path pairs
+ h := sha256.New()
+ h.Write([]byte(strings.Join(checksumPaths, "\n")))
+ return h.Sum(nil), nil
+}
diff --git a/docker/devbox-bundled/bootstrap/internal/utils/checksum_test.go b/docker/devbox-bundled/bootstrap/internal/utils/checksum_test.go
new file mode 100644
index 00000000000..049547d8727
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/utils/checksum_test.go
@@ -0,0 +1,37 @@
+package utils
+
+import (
+ "encoding/hex"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestFileChecksum(t *testing.T) {
+ c, err := FileChecksum(filepath.Join("testdata/foo"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(
+ t,
+ hex.EncodeToString(c),
+ "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c",
+ "checksums should match",
+ )
+}
+
+func TestFileCollectionChecksum(t *testing.T) {
+ c, err := FileCollectionChecksum(
+ []string{filepath.Join("testdata/foo"), filepath.Join("testdata/bar")},
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(
+ t,
+ hex.EncodeToString(c),
+ "2db7bade901cc7260437c7cdd91b9f0cdb34fe3e5c72c5d0e756f3ce7d0c3f6a",
+ "checksums should match",
+ )
+}
diff --git a/docker/devbox-bundled/bootstrap/internal/utils/patch.go b/docker/devbox-bundled/bootstrap/internal/utils/patch.go
new file mode 100644
index 00000000000..319ae9d9c39
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/utils/patch.go
@@ -0,0 +1,53 @@
+package utils
+
+import (
+ "encoding/json"
+ "reflect"
+
+ "k8s.io/apimachinery/pkg/runtime"
+ "sigs.k8s.io/yaml"
+)
+
+func sanitizePatch(m map[string]interface{}, filterNil, filterEmpty bool) {
+ val := reflect.ValueOf(m)
+ for _, e := range val.MapKeys() {
+ v := val.MapIndex(e)
+ if v.IsNil() {
+ if filterNil {
+ delete(m, e.String())
+ }
+ continue
+ }
+ switch t := v.Interface().(type) {
+ case map[string]interface{}:
+ sanitizePatch(t, filterNil, filterEmpty)
+ if filterEmpty && len(t) == 0 {
+ delete(m, e.String())
+ }
+ case []interface{}:
+ if filterEmpty && len(t) == 0 {
+ delete(m, e.String())
+ }
+ }
+ }
+}
+
+func MarshalPatch(o interface{}, filterNil, filterEmpty bool) ([]byte, error) {
+ m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(o)
+ if err != nil {
+ return nil, err
+ }
+ sanitizePatch(m, filterNil, filterEmpty)
+
+ j, err := json.Marshal(m)
+ if err != nil {
+ return nil, err
+ }
+
+ y, err := yaml.JSONToYAML(j)
+ if err != nil {
+ return nil, err
+ }
+
+ return y, nil
+}
diff --git a/docker/devbox-bundled/bootstrap/internal/utils/patch_test.go b/docker/devbox-bundled/bootstrap/internal/utils/patch_test.go
new file mode 100644
index 00000000000..a757d15a259
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/utils/patch_test.go
@@ -0,0 +1,115 @@
+package utils
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ appsv1 "k8s.io/api/apps/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestMarshalPatch(t *testing.T) {
+ patch := appsv1.Deployment{
+ TypeMeta: metav1.TypeMeta{
+ Kind: "Deployment",
+ APIVersion: "apps/v1",
+ },
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "foo",
+ Namespace: "bar",
+ },
+ Spec: appsv1.DeploymentSpec{},
+ }
+ t.Run("NoFilter", func(t *testing.T) {
+ y, err := MarshalPatch(&patch, false, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(
+ t,
+ string(y),
+ `apiVersion: apps/v1
+kind: Deployment
+metadata:
+ creationTimestamp: null
+ name: foo
+ namespace: bar
+spec:
+ selector: null
+ strategy: {}
+ template:
+ metadata:
+ creationTimestamp: null
+ spec:
+ containers: null
+status: {}
+`,
+ "YAML strings should match",
+ )
+ })
+ t.Run("FilterNil", func(t *testing.T) {
+ y, err := MarshalPatch(&patch, true, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(
+ t,
+ string(y),
+ `apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: foo
+ namespace: bar
+spec:
+ strategy: {}
+ template:
+ metadata: {}
+ spec: {}
+status: {}
+`,
+ "YAML strings should match",
+ )
+ })
+ t.Run("FilterEmpty", func(t *testing.T) {
+ y, err := MarshalPatch(&patch, false, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(
+ t,
+ string(y),
+ `apiVersion: apps/v1
+kind: Deployment
+metadata:
+ creationTimestamp: null
+ name: foo
+ namespace: bar
+spec:
+ selector: null
+ template:
+ metadata:
+ creationTimestamp: null
+ spec:
+ containers: null
+`,
+ "YAML strings should match",
+ )
+ })
+ t.Run("FilterAll", func(t *testing.T) {
+ y, err := MarshalPatch(&patch, true, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assert.Equal(
+ t,
+ string(y),
+ `apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: foo
+ namespace: bar
+`,
+ "YAML strings should match",
+ )
+ })
+}
diff --git a/docker/devbox-bundled/bootstrap/internal/utils/testdata/bar b/docker/devbox-bundled/bootstrap/internal/utils/testdata/bar
new file mode 100644
index 00000000000..5716ca5987c
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/utils/testdata/bar
@@ -0,0 +1 @@
+bar
diff --git a/docker/devbox-bundled/bootstrap/internal/utils/testdata/foo b/docker/devbox-bundled/bootstrap/internal/utils/testdata/foo
new file mode 100644
index 00000000000..257cc5642cb
--- /dev/null
+++ b/docker/devbox-bundled/bootstrap/internal/utils/testdata/foo
@@ -0,0 +1 @@
+foo
diff --git a/docker/devbox-bundled/containerd-config.toml.tmpl b/docker/devbox-bundled/containerd-config.toml.tmpl
new file mode 100644
index 00000000000..7cda384aa08
--- /dev/null
+++ b/docker/devbox-bundled/containerd-config.toml.tmpl
@@ -0,0 +1,10 @@
+{{ template "base" . }}
+
+# Override: make the NVIDIA runtime the default. k3s auto-registers a
+# `nvidia` runtime at startup when /usr/bin/nvidia-container-runtime is
+# present. By switching the default, pods requesting `nvidia.com/gpu` get
+# GPU access without needing `runtimeClassName: nvidia` in their spec.
+# nvidia-container-runtime is a passthrough when no GPU is requested, so
+# non-GPU pods are unaffected.
+[plugins.'io.containerd.cri.v1.runtime'.containerd]
+ default_runtime_name = "nvidia"
diff --git a/docker/devbox-bundled/images/manifest.txt b/docker/devbox-bundled/images/manifest.txt
new file mode 100644
index 00000000000..e472a21c60e
--- /dev/null
+++ b/docker/devbox-bundled/images/manifest.txt
@@ -0,0 +1,8 @@
+docker.io/bitnami/os-shell:sandbox=bitnamilegacy/os-shell:11-debian-11
+docker.io/library/registry:sandbox=registry:2.8.1
+docker.io/rancher/local-path-provisioner:v0.0.21
+docker.io/rancher/mirrored-coredns-coredns:1.9.1
+docker.io/rancher/mirrored-library-busybox:1.34.1
+docker.io/rancher/mirrored-pause:3.6
+docker.io/rustfs/rustfs:sandbox=rustfs/rustfs:latest
+docker.io/unionai-oss/flyteconsole-v2:sandbox=ghcr.io/unionai-oss/flyteconsole-v2:latest
diff --git a/docker/devbox-bundled/images/preload b/docker/devbox-bundled/images/preload
new file mode 100755
index 00000000000..fac95ee3d5d
--- /dev/null
+++ b/docker/devbox-bundled/images/preload
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -ex
+
+__save_image() {
+ local dest="$1"
+ local tar="$(echo ${dest} | cut -d: -f1 | tr / -).tar"
+ local src="$( [ -n "$2" ] && echo "$2" || echo ${dest} )"
+ podman pull --arch ${TARGETARCH} ${src}
+ podman tag ${src} ${dest}
+ podman save -o ${tar} ${dest}
+}
+
+MANIFEST="$(realpath "$1")"
+mkdir -p images
+cd images
+while read line; do
+ __save_image $(echo ${line} | tr = ' ')
+done < ${MANIFEST}
diff --git a/docker/devbox-bundled/kustomize/complete/kustomization.yaml b/docker/devbox-bundled/kustomize/complete/kustomization.yaml
new file mode 100644
index 00000000000..c6db3229b0a
--- /dev/null
+++ b/docker/devbox-bundled/kustomize/complete/kustomization.yaml
@@ -0,0 +1,30 @@
+helmGlobals:
+ chartHome: ../../../../charts
+helmCharts:
+- name: flyte-devbox
+ releaseName: flyte-devbox
+ namespace: flyte
+ valuesInline:
+ # Disabled: PostgreSQL runs as an embedded process on the host via
+ # the fergusstrange/embedded-postgres library instead of a pod.
+ postgresql:
+ enabled: false
+ flyte-binary:
+ deployment:
+ waitForDB:
+ image:
+ repository: rancher/mirrored-library-busybox
+ tag: "1.34.1"
+ command:
+ - sh
+ - -ec
+ args:
+ - |
+ until nc -z postgresql.flyte 5432; do
+ echo waiting for database
+ sleep 0.1
+ done
+namespace: flyte
+resources:
+- ../namespace.yaml
+- ../embedded-postgres-service.yaml
diff --git a/docker/devbox-bundled/kustomize/dev/kustomization.yaml b/docker/devbox-bundled/kustomize/dev/kustomization.yaml
new file mode 100644
index 00000000000..f379d0604d1
--- /dev/null
+++ b/docker/devbox-bundled/kustomize/dev/kustomization.yaml
@@ -0,0 +1,20 @@
+helmGlobals:
+ chartHome: ../../../../charts
+helmCharts:
+- name: flyte-devbox
+ releaseName: flyte-devbox
+ namespace: flyte
+ valuesInline:
+ flyte-binary:
+ enabled: false
+ # Disabled: PostgreSQL runs as an embedded process on the host via
+ # the fergusstrange/embedded-postgres library instead of a pod.
+ postgresql:
+ enabled: false
+ sandbox:
+ dev: true
+namespace: flyte
+resources:
+- ../namespace.yaml
+- ../../../../charts/flyte-binary/templates/crds/flyte.org_taskactions.yaml
+- ../embedded-postgres-service.yaml
diff --git a/docker/devbox-bundled/kustomize/embedded-postgres-service.yaml b/docker/devbox-bundled/kustomize/embedded-postgres-service.yaml
new file mode 100644
index 00000000000..c6e050b5ebd
--- /dev/null
+++ b/docker/devbox-bundled/kustomize/embedded-postgres-service.yaml
@@ -0,0 +1,29 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: postgresql
+ namespace: flyte
+ labels:
+ app.kubernetes.io/name: embedded-postgresql
+spec:
+ type: NodePort
+ ports:
+ - name: tcp-postgresql
+ port: 5432
+ targetPort: 5432
+ nodePort: 30001
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+ name: postgresql
+ namespace: flyte
+ labels:
+ app.kubernetes.io/name: embedded-postgresql
+subsets:
+ - addresses:
+ - ip: "%{NODE_IP}%"
+ ports:
+ - name: tcp-postgresql
+ port: 5432
diff --git a/docker/devbox-bundled/kustomize/minio-patch.yaml b/docker/devbox-bundled/kustomize/minio-patch.yaml
new file mode 100644
index 00000000000..2f8b26f6782
--- /dev/null
+++ b/docker/devbox-bundled/kustomize/minio-patch.yaml
@@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: minio
+ namespace: flyte
+spec:
+ template:
+ spec:
+ initContainers:
+ - name: volume-permissions
+ command:
+ - /bin/bash
+ - -ec
+ - |
+ chown -R 1001:1001 /data
+ mkdir -p /data/flyte-data
+ chown 1001:1001 /data/flyte-data
+ containers:
+ - name: minio
+ command:
+ - minio
+ - server
+ - /data
+ - --console-address
+ - :9001
diff --git a/docker/devbox-bundled/kustomize/namespace.yaml b/docker/devbox-bundled/kustomize/namespace.yaml
new file mode 100644
index 00000000000..ca27d7f8850
--- /dev/null
+++ b/docker/devbox-bundled/kustomize/namespace.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: flyte
diff --git a/docker/devbox-bundled/manifests/complete.yaml b/docker/devbox-bundled/manifests/complete.yaml
new file mode 100644
index 00000000000..f8f9c31b31a
--- /dev/null
+++ b/docker/devbox-bundled/manifests/complete.yaml
@@ -0,0 +1,1208 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: flyte
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+ annotations:
+ controller-gen.kubebuilder.io/version: v0.19.0
+ name: taskactions.flyte.org
+spec:
+ group: flyte.org
+ names:
+ kind: TaskAction
+ listKind: TaskActionList
+ plural: taskactions
+ singular: taskaction
+ scope: Namespaced
+ versions:
+ - additionalPrinterColumns:
+ - jsonPath: .spec.runName
+ name: Run
+ type: string
+ - jsonPath: .spec.actionName
+ name: Action
+ type: string
+ - jsonPath: .spec.taskType
+ name: TaskType
+ type: string
+ - jsonPath: .status.conditions[?(@.type=='Progressing')].reason
+ name: Status
+ type: string
+ - jsonPath: .metadata.creationTimestamp
+ name: Age
+ type: date
+ - jsonPath: .status.conditions[?(@.type=='Progressing')].status
+ name: Progressing
+ priority: 1
+ type: string
+ - jsonPath: .status.conditions[?(@.type=='Succeeded')].status
+ name: Succeeded
+ priority: 1
+ type: string
+ - jsonPath: .status.conditions[?(@.type=='Failed')].status
+ name: Failed
+ priority: 1
+ type: string
+ name: v1
+ schema:
+ openAPIV3Schema:
+ description: TaskAction is the Schema for the taskactions API
+ properties:
+ apiVersion:
+ description: |-
+ APIVersion defines the versioned schema of this representation of an object.
+ Servers should convert recognized schemas to the latest internal value, and
+ may reject unrecognized values.
+ More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+ type: string
+ kind:
+ description: |-
+ Kind is a string value representing the REST resource this object represents.
+ Servers may infer this from the endpoint the client submits requests to.
+ Cannot be updated.
+ In CamelCase.
+ More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+ type: string
+ metadata:
+ type: object
+ spec:
+ description: spec defines the desired state of TaskAction
+ properties:
+ actionName:
+ description: ActionName is the unique name of this action within the
+ run
+ maxLength: 30
+ minLength: 1
+ type: string
+ cacheKey:
+ description: |-
+ CacheKey enables cache lookup/writeback for this task action when set.
+ This is propagated from workflow.TaskAction.cache_key.
+ maxLength: 256
+ type: string
+ domain:
+ description: Domain this action belongs to
+ maxLength: 63
+ minLength: 1
+ type: string
+ envVars:
+ additionalProperties:
+ type: string
+ description: EnvVars are run-scoped environment variables projected
+ from RunSpec for executor runtime use.
+ type: object
+ group:
+ description: Group is the group this action belongs to, if applicable.
+ maxLength: 256
+ type: string
+ inputUri:
+ description: InputURI is the path to the input data for this action
+ minLength: 1
+ type: string
+ interruptible:
+ description: Interruptible is the run-scoped interruptibility override
+ projected from RunSpec.
+ type: boolean
+ parentActionName:
+ description: ParentActionName is the optional name of the parent action
+ maxLength: 30
+ minLength: 1
+ type: string
+ project:
+ description: Project this action belongs to
+ maxLength: 63
+ minLength: 1
+ type: string
+ runName:
+ description: RunName is the name of the run this action belongs to
+ maxLength: 30
+ minLength: 1
+ type: string
+ runOutputBase:
+ description: RunOutputBase is the base path where this action should
+ write its output
+ minLength: 1
+ type: string
+ shortName:
+ description: ShortName is the human-readable display name for this
+ task
+ maxLength: 63
+ type: string
+ taskTemplate:
+ description: TaskTemplate is the proto-serialized core.TaskTemplate
+ stored inline in etcd
+ format: byte
+ type: string
+ taskType:
+ description: TaskType identifies which plugin handles this task (e.g.
+ "container", "spark", "ray")
+ maxLength: 63
+ minLength: 1
+ type: string
+ required:
+ - actionName
+ - domain
+ - inputUri
+ - project
+ - runName
+ - runOutputBase
+ - taskTemplate
+ - taskType
+ type: object
+ status:
+ description: status defines the observed state of TaskAction
+ properties:
+ attempts:
+ description: Attempts is the latest observed action attempt number,
+ starting from 1.
+ format: int32
+ type: integer
+ cacheStatus:
+ description: CacheStatus is the latest observed cache lookup result
+ for this action.
+ format: int32
+ type: integer
+ conditions:
+ description: |-
+ conditions represent the current state of the TaskAction resource.
+ Each condition has a unique type and reflects the status of a specific aspect of the resource.
+
+ Standard condition types include:
+ - "Available": the resource is fully functional
+ - "Progressing": the resource is being created or updated
+ - "Degraded": the resource failed to reach or maintain its desired state
+
+ The status of each condition is one of True, False, or Unknown.
+ items:
+ description: Condition contains details for one aspect of the current
+ state of this API Resource.
+ properties:
+ lastTransitionTime:
+ description: |-
+ lastTransitionTime is the last time the condition transitioned from one status to another.
+ This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
+ format: date-time
+ type: string
+ message:
+ description: |-
+ message is a human readable message indicating details about the transition.
+ This may be an empty string.
+ maxLength: 32768
+ type: string
+ observedGeneration:
+ description: |-
+ observedGeneration represents the .metadata.generation that the condition was set based upon.
+ For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+ with respect to the current state of the instance.
+ format: int64
+ minimum: 0
+ type: integer
+ reason:
+ description: |-
+ reason contains a programmatic identifier indicating the reason for the condition's last transition.
+ Producers of specific condition types may define expected values and meanings for this field,
+ and whether the values are considered a guaranteed API.
+ The value should be a CamelCase string.
+ This field may not be empty.
+ maxLength: 1024
+ minLength: 1
+ pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+ type: string
+ status:
+ description: status of the condition, one of True, False, Unknown.
+ enum:
+ - "True"
+ - "False"
+ - Unknown
+ type: string
+ type:
+ description: type of condition in CamelCase or in foo.example.com/CamelCase.
+ maxLength: 316
+ pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+ type: string
+ required:
+ - lastTransitionTime
+ - message
+ - reason
+ - status
+ - type
+ type: object
+ type: array
+ x-kubernetes-list-map-keys:
+ - type
+ x-kubernetes-list-type: map
+ phaseHistory:
+ description: |-
+ PhaseHistory is an append-only log of phase transitions. Unlike conditions
+ (which are updated in-place by type), this preserves the full timeline:
+ Queued → Initializing → Executing → Succeeded/Failed, each with a timestamp.
+ items:
+ description: PhaseTransition records a phase change with its timestamp.
+ properties:
+ message:
+ description: Message is an optional human-readable message about
+ the transition.
+ type: string
+ occurredAt:
+ description: OccurredAt is when this phase transition happened.
+ format: date-time
+ type: string
+ phase:
+ description: Phase is the phase that was entered (e.g. "Queued",
+ "Initializing", "Executing", "Succeeded", "Failed").
+ type: string
+ required:
+ - occurredAt
+ - phase
+ type: object
+ type: array
+ pluginPhase:
+ description: PluginPhase is a human-readable representation of the
+ plugin's current phase.
+ type: string
+ pluginPhaseVersion:
+ description: PluginPhaseVersion is the version of the current plugin
+ phase.
+ format: int32
+ type: integer
+ pluginState:
+ description: PluginState is the Gob-encoded plugin state from the
+ last reconciliation round.
+ format: byte
+ type: string
+ pluginStateVersion:
+ description: PluginStateVersion tracks the version of the plugin state
+ schema for compatibility.
+ type: integer
+ stateJson:
+ description: StateJSON is the JSON serialized NodeStatus that was
+ last sent to the State Service
+ type: string
+ type: object
+ required:
+ - spec
+ type: object
+ served: true
+ storage: true
+ subresources:
+ status: {}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-binary
+ helm.sh/chart: flyte-binary-v0.2.0
+ name: flyte-binary
+ namespace: flyte
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-binary
+ helm.sh/chart: flyte-binary-v0.2.0
+ name: flyte-binary-cluster-role
+ namespace: flyte
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - pods
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+- apiGroups:
+ - ""
+ resources:
+ - events
+ verbs:
+ - create
+ - delete
+ - patch
+ - update
+- apiGroups:
+ - flyte.org
+ resources:
+ - taskactions
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+- apiGroups:
+ - admissionregistration.k8s.io
+ resources:
+ - mutatingwebhookconfigurations
+ verbs:
+ - create
+ - get
+ - list
+ - patch
+ - update
+ - watch
+- apiGroups:
+ - ""
+ resources:
+ - secrets
+ verbs:
+ - create
+ - get
+ - update
+- apiGroups:
+ - '*'
+ resources:
+ - '*'
+ verbs:
+ - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-binary
+ helm.sh/chart: flyte-binary-v0.2.0
+ name: flyte-binary-cluster-role-binding
+ namespace: flyte
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: flyte-binary-cluster-role
+subjects:
+- kind: ServiceAccount
+ name: flyte-binary
+ namespace: flyte
+---
+apiVersion: v1
+data:
+ config.yml: |-
+ health:
+ storagedriver:
+ enabled: true
+ interval: 10s
+ threshold: 3
+ http:
+ addr: :5000
+ debug:
+ addr: :5001
+ prometheus:
+ enabled: false
+ path: /metrics
+ headers:
+ X-Content-Type-Options:
+ - nosniff
+ log:
+ fields:
+ service: registry
+ storage:
+ cache:
+ blobdescriptor: inmemory
+ version: 0.1
+kind: ConfigMap
+metadata:
+ labels:
+ app: docker-registry
+ chart: docker-registry-2.2.2
+ heritage: Helm
+ release: flyte-devbox
+ name: docker-registry-config
+ namespace: flyte
+---
+apiVersion: v1
+data:
+ 000-core.yaml: |
+ logger:
+ show-source: true
+ level: 5
+ webhook:
+ certDir: /var/run/flyte/certs
+ localCert: true
+ secretName: flyte-binary-webhook-secret
+ serviceName: flyte-binary-webhook
+ servicePort: 443
+
+ actions:
+ kubernetes:
+ kubeconfig: ""
+ namespace: flyte
+ watchBufferSize: 100
+ dataproxy:
+ download:
+ maxExpiresIn: 1h
+ upload:
+ defaultFileNameLength: 20
+ maxExpiresIn: 1h
+ maxSize: 100Mi
+ storagePrefix: uploads
+ manager:
+ executor:
+ healthProbePort: 8081
+ kubernetes:
+ burst: 2000
+ kubeconfig: ""
+ namespace: flyte
+ qps: 1000
+ timeout: 30s
+ server:
+ host: 0.0.0.0
+ port: 8090
+ runs:
+ database:
+ connMaxLifeTime: 1h
+ maxIdleConnections: 10
+ maxOpenConnections: 100
+ postgres:
+ dbname: flyte
+ debug: false
+ host: 127.0.0.1
+ options: sslmode=disable
+ password: ""
+ port: 5432
+ username: postgres
+ server:
+ host: 0.0.0.0
+ port: 8090
+ storagePrefix: s3://flyte-data
+ watchBufferSize: 100
+ secret:
+ kubernetes:
+ burst: 200
+ clusterName: flyte-devbox
+ kubeconfig: ""
+ namespace: flyte
+ qps: 100
+ timeout: 30s
+ 001-plugins.yaml: |
+ tasks:
+ task-plugins:
+ default-for-task-types:
+ container: container
+ container_array: k8s-array
+ sidecar: sidecar
+ enabled-plugins:
+ - container
+ - sidecar
+ - connector-service
+ - echo
+ plugins:
+ logs:
+ kubernetes-enabled: false
+ cloudwatch-enabled: false
+ stackdriver-enabled: false
+ k8s:
+ co-pilot:
+ image: "cr.flyte.org/flyteorg/flytecopilot:v1.16.4"
+ k8s-array:
+ logs:
+ config:
+ kubernetes-enabled: false
+ cloudwatch-enabled: false
+ stackdriver-enabled: false
+ 002-database.yaml: |
+ database:
+ postgres:
+ username: postgres
+ host: postgresql
+ port: 5432
+ dbname: flyte
+ options: "sslmode=disable"
+ 003-storage.yaml: |
+ storage:
+ type: stow
+ stow:
+ kind: s3
+ config:
+ region: us-east-1
+ disable_ssl: true
+ v2_signing: true
+ endpoint: http://rustfs.flyte:9000
+ auth_type: accesskey
+ container: flyte-data
+ 100-inline-config.yaml: |
+ plugins:
+ k8s:
+ default-env-vars:
+ - FLYTE_AWS_ENDPOINT: http://rustfs.flyte:9000
+ - FLYTE_AWS_ACCESS_KEY_ID: rustfs
+ - FLYTE_AWS_SECRET_ACCESS_KEY: rustfsstorage
+ - _U_EP_OVERRIDE: flyte-binary-http.flyte:8090
+ - _U_INSECURE: "true"
+ - _U_USE_ACTIONS: "1"
+ runs:
+ database:
+ postgres:
+ dbName: runs
+ host: postgresql.flyte
+ password: postgres
+ port: 5432
+ user: postgres
+ storage:
+ signedURL:
+ stowConfigOverride:
+ endpoint: http://localhost:30002
+ task_resources:
+ defaults:
+ cpu: 500m
+ ephemeralStorage: 0
+ gpu: 0
+ memory: 1Gi
+ limits:
+ cpu: 0
+ ephemeralStorage: 0
+ gpu: 0
+ memory: 0
+kind: ConfigMap
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-binary
+ helm.sh/chart: flyte-binary-v0.2.0
+ name: flyte-binary-config
+ namespace: flyte
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flyte-devbox-extra-cluster-resource-templates
+ namespace: flyte
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: flyte-devbox-extra-config
+ namespace: flyte
+---
+apiVersion: v1
+data:
+ haSharedSecret: Zmx5dGVzYW5kYm94c2VjcmV0
+ proxyPassword: ""
+ proxyUsername: ""
+kind: Secret
+metadata:
+ labels:
+ app: docker-registry
+ chart: docker-registry-2.2.2
+ heritage: Helm
+ release: flyte-devbox
+ name: docker-registry-secret
+ namespace: flyte
+type: Opaque
+---
+apiVersion: v1
+kind: Secret
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-binary
+ helm.sh/chart: flyte-binary-v0.2.0
+ name: flyte-binary-config-secret
+ namespace: flyte
+stringData:
+ 012-database-secrets.yaml: |
+ database:
+ postgres:
+ password: "postgres"
+ 013-storage-secrets.yaml: |
+ storage:
+ stow:
+ config:
+ access_key_id: "rustfs"
+ secret_key: "rustfsstorage"
+type: Opaque
+---
+apiVersion: v1
+data:
+ access-key: cnVzdGZz
+ secret-key: cnVzdGZzc3RvcmFnZQ==
+kind: Secret
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: rustfs
+ namespace: flyte
+type: Opaque
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+ labels:
+ app.kubernetes.io/name: embedded-postgresql
+ name: postgresql
+ namespace: flyte
+subsets:
+- addresses:
+ - ip: '%{NODE_IP}%'
+ ports:
+ - name: tcp-postgresql
+ port: 5432
+---
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app: docker-registry
+ chart: docker-registry-2.2.2
+ heritage: Helm
+ release: flyte-devbox
+ name: docker-registry
+ namespace: flyte
+spec:
+ ports:
+ - name: http-5000
+ nodePort: 30000
+ port: 5000
+ protocol: TCP
+ targetPort: 5000
+ selector:
+ app: docker-registry
+ release: flyte-devbox
+ type: NodePort
+---
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-binary
+ helm.sh/chart: flyte-binary-v0.2.0
+ name: flyte-binary-http
+ namespace: flyte
+spec:
+ ports:
+ - name: http
+ nodePort: null
+ port: 8090
+ targetPort: http
+ selector:
+ app.kubernetes.io/component: flyte-binary
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: flyte-binary
+ type: ClusterIP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-binary
+ helm.sh/chart: flyte-binary-v0.2.0
+ name: flyte-binary-webhook
+ namespace: flyte
+spec:
+ ports:
+ - name: webhook
+ port: 443
+ protocol: TCP
+ targetPort: 9443
+ selector:
+ app.kubernetes.io/component: flyte-binary
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: flyte-binary
+---
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: flyte-console
+ namespace: flyte
+spec:
+ ports:
+ - name: http
+ port: 80
+ protocol: TCP
+ targetPort: http
+ selector:
+ app.kubernetes.io/component: console
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: flyte-devbox
+---
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/name: embedded-postgresql
+ name: postgresql
+ namespace: flyte
+spec:
+ ports:
+ - name: tcp-postgresql
+ nodePort: 30001
+ port: 5432
+ targetPort: 5432
+ type: NodePort
+---
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: rustfs
+ namespace: flyte
+spec:
+ ports:
+ - name: rustfs-api
+ nodePort: 30002
+ port: 9000
+ targetPort: rustfs-api
+ selector:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: rustfs
+ type: NodePort
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: flyte-devbox-rustfs-storage
+ namespace: flyte
+spec:
+ accessModes:
+ - ReadWriteOnce
+ capacity:
+ storage: 1Gi
+ hostPath:
+ path: /var/lib/flyte/storage/rustfs
+ storageClassName: manual
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: flyte-devbox-rustfs-storage
+ namespace: flyte
+spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ storageClassName: manual
+ volumeName: flyte-devbox-rustfs-storage
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ labels:
+ app: docker-registry
+ chart: docker-registry-2.2.2
+ heritage: Helm
+ release: flyte-devbox
+ name: docker-registry
+ namespace: flyte
+spec:
+ minReadySeconds: 5
+ replicas: 1
+ selector:
+ matchLabels:
+ app: docker-registry
+ release: flyte-devbox
+ template:
+ metadata:
+ annotations:
+ checksum/config: 6b06ba712a995bdcc044732aa5e5b9d56ce24061c6ad5c2b7f35508a00c7863b
+ checksum/secret: 8f6b14d94a45833195baa1b863e43296aa157e65ef50c983e672c34648d36c50
+ labels:
+ app: docker-registry
+ release: flyte-devbox
+ spec:
+ containers:
+ - command:
+ - /bin/registry
+ - serve
+ - /etc/docker/registry/config.yml
+ env:
+ - name: REGISTRY_HTTP_SECRET
+ valueFrom:
+ secretKeyRef:
+ key: haSharedSecret
+ name: docker-registry-secret
+ - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
+ value: /var/lib/registry
+ image: registry:sandbox
+ imagePullPolicy: Never
+ livenessProbe:
+ httpGet:
+ path: /
+ port: 5000
+ name: docker-registry
+ ports:
+ - containerPort: 5000
+ readinessProbe:
+ httpGet:
+ path: /
+ port: 5000
+ resources: {}
+ volumeMounts:
+ - mountPath: /etc/docker/registry
+ name: docker-registry-config
+ - mountPath: /var/lib/registry/
+ name: data
+ securityContext:
+ fsGroup: 1000
+ runAsUser: 1000
+ volumes:
+ - configMap:
+ name: docker-registry-config
+ name: docker-registry-config
+ - emptyDir: {}
+ name: data
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-binary
+ helm.sh/chart: flyte-binary-v0.2.0
+ name: flyte-binary
+ namespace: flyte
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app.kubernetes.io/component: flyte-binary
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: flyte-binary
+ strategy:
+ type: Recreate
+ template:
+ metadata:
+ annotations:
+ checksum/configuration: 646d21c3d7efb87ed1de45515bf092b575b364810dd9264a2d9b2e7b26ddbbcb
+ checksum/configuration-secret: e70194084619f4a1d4017093aac6367047167107fd0222513a32a61734629cac
+ labels:
+ app.kubernetes.io/component: flyte-binary
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: flyte-binary
+ spec:
+ containers:
+ - args:
+ - --config
+ - /etc/flyte/config.d/*.yaml
+ env:
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ image: flyte-binary-v2:sandbox
+ imagePullPolicy: Never
+ livenessProbe:
+ httpGet:
+ path: /healthz
+ port: http
+ name: flyte
+ ports:
+ - containerPort: 8090
+ name: http
+ - containerPort: 9443
+ name: webhook
+ readinessProbe:
+ httpGet:
+ path: /readyz
+ port: http
+ periodSeconds: 1
+ startupProbe:
+ failureThreshold: 30
+ httpGet:
+ path: /healthz
+ port: http
+ periodSeconds: 1
+ volumeMounts:
+ - mountPath: /etc/flyte/config.d
+ name: config
+ - mountPath: /var/run/flyte/certs
+ name: webhook-certs
+ readOnly: false
+ initContainers:
+ - args:
+ - |
+ until nc -z postgresql.flyte 5432; do
+ echo waiting for database
+ sleep 0.1
+ done
+ command:
+ - sh
+ - -ec
+ image: rancher/mirrored-library-busybox:1.34.1
+ imagePullPolicy: Never
+ name: wait-for-db
+ serviceAccountName: flyte-binary
+ volumes:
+ - name: config
+ projected:
+ sources:
+ - configMap:
+ name: flyte-binary-config
+ - secret:
+ name: flyte-binary-config-secret
+ - configMap:
+ name: flyte-devbox-extra-config
+ - emptyDir: {}
+ name: webhook-certs
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: flyte-console
+ namespace: flyte
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app.kubernetes.io/component: console
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: flyte-devbox
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/component: console
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: flyte-devbox
+ spec:
+ containers:
+ - image: docker.io/unionai-oss/flyteconsole-v2:sandbox
+ imagePullPolicy: Never
+ livenessProbe:
+ httpGet:
+ path: /v2
+ port: http
+ initialDelaySeconds: 5
+ periodSeconds: 30
+ name: console
+ ports:
+ - containerPort: 8080
+ name: http
+ protocol: TCP
+ readinessProbe:
+ httpGet:
+ path: /v2
+ port: http
+ initialDelaySeconds: 5
+ periodSeconds: 10
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: rustfs
+ namespace: flyte
+spec:
+ selector:
+ matchLabels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: rustfs
+ strategy:
+ type: Recreate
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: rustfs
+ spec:
+ containers:
+ - env:
+ - name: RUSTFS_ADDRESS
+ value: 0.0.0.0:9000
+ - name: RUSTFS_VOLUMES
+ value: /data
+ - name: RUSTFS_ACCESS_KEY
+ valueFrom:
+ secretKeyRef:
+ key: access-key
+ name: rustfs
+ - name: RUSTFS_SECRET_KEY
+ valueFrom:
+ secretKeyRef:
+ key: secret-key
+ name: rustfs
+ image: rustfs/rustfs:sandbox
+ imagePullPolicy: Never
+ livenessProbe:
+ failureThreshold: 5
+ initialDelaySeconds: 10
+ periodSeconds: 10
+ tcpSocket:
+ port: rustfs-api
+ name: rustfs
+ ports:
+ - containerPort: 9000
+ name: rustfs-api
+ protocol: TCP
+ readinessProbe:
+ failureThreshold: 5
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ tcpSocket:
+ port: rustfs-api
+ securityContext:
+ runAsNonRoot: true
+ runAsUser: 10001
+ volumeMounts:
+ - mountPath: /data
+ name: data
+ initContainers:
+ - command:
+ - /bin/sh
+ - -ec
+ - |
+ chown -R 10001:10001 /data
+ mkdir -p /data/flyte-data
+ chown 10001:10001 /data/flyte-data
+ image: busybox:latest
+ imagePullPolicy: IfNotPresent
+ name: volume-permissions
+ securityContext:
+ runAsUser: 0
+ volumeMounts:
+ - mountPath: /data
+ name: data
+ securityContext:
+ fsGroup: 10001
+ volumes:
+ - name: data
+ persistentVolumeClaim:
+ claimName: flyte-devbox-rustfs-storage
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: traefik
+ namespace: kube-system
+spec:
+ valuesContent: |
+ service:
+ type: NodePort
+ ports:
+ web:
+ nodePort: 30080
+ transport:
+ respondingTimeouts:
+ readTimeout: 0
+ websecure:
+ expose: false
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: flyte-console
+ namespace: flyte
+spec:
+ rules:
+ - http:
+ paths:
+ - backend:
+ service:
+ name: flyte-console
+ port:
+ number: 80
+ path: /v2
+ pathType: Prefix
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: flyte-devbox-api
+ namespace: flyte
+spec:
+ rules:
+ - http:
+ paths:
+ - backend:
+ service:
+ name: flyte-binary-http
+ port:
+ number: 8090
+ path: /healthz
+ pathType: Exact
+ - backend:
+ service:
+ name: flyte-binary-http
+ port:
+ number: 8090
+ path: /readyz
+ pathType: Exact
+ - backend:
+ service:
+ name: flyte-binary-http
+ port:
+ number: 8090
+ path: /flyteidl2.
+ pathType: Prefix
diff --git a/docker/devbox-bundled/manifests/dev.yaml b/docker/devbox-bundled/manifests/dev.yaml
new file mode 100644
index 00000000000..f6d4bda06ea
--- /dev/null
+++ b/docker/devbox-bundled/manifests/dev.yaml
@@ -0,0 +1,822 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: flyte
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+ annotations:
+ controller-gen.kubebuilder.io/version: v0.19.0
+ name: taskactions.flyte.org
+spec:
+ group: flyte.org
+ names:
+ kind: TaskAction
+ listKind: TaskActionList
+ plural: taskactions
+ singular: taskaction
+ scope: Namespaced
+ versions:
+ - additionalPrinterColumns:
+ - jsonPath: .spec.runName
+ name: Run
+ type: string
+ - jsonPath: .spec.actionName
+ name: Action
+ type: string
+ - jsonPath: .spec.taskType
+ name: TaskType
+ type: string
+ - jsonPath: .status.conditions[?(@.type=='Progressing')].reason
+ name: Status
+ type: string
+ - jsonPath: .metadata.creationTimestamp
+ name: Age
+ type: date
+ - jsonPath: .status.conditions[?(@.type=='Progressing')].status
+ name: Progressing
+ priority: 1
+ type: string
+ - jsonPath: .status.conditions[?(@.type=='Succeeded')].status
+ name: Succeeded
+ priority: 1
+ type: string
+ - jsonPath: .status.conditions[?(@.type=='Failed')].status
+ name: Failed
+ priority: 1
+ type: string
+ name: v1
+ schema:
+ openAPIV3Schema:
+ description: TaskAction is the Schema for the taskactions API
+ properties:
+ apiVersion:
+ description: |-
+ APIVersion defines the versioned schema of this representation of an object.
+ Servers should convert recognized schemas to the latest internal value, and
+ may reject unrecognized values.
+ More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+ type: string
+ kind:
+ description: |-
+ Kind is a string value representing the REST resource this object represents.
+ Servers may infer this from the endpoint the client submits requests to.
+ Cannot be updated.
+ In CamelCase.
+ More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+ type: string
+ metadata:
+ type: object
+ spec:
+ description: spec defines the desired state of TaskAction
+ properties:
+ actionName:
+ description: ActionName is the unique name of this action within the
+ run
+ maxLength: 30
+ minLength: 1
+ type: string
+ cacheKey:
+ description: |-
+ CacheKey enables cache lookup/writeback for this task action when set.
+ This is propagated from workflow.TaskAction.cache_key.
+ maxLength: 256
+ type: string
+ domain:
+ description: Domain this action belongs to
+ maxLength: 63
+ minLength: 1
+ type: string
+ envVars:
+ additionalProperties:
+ type: string
+ description: EnvVars are run-scoped environment variables projected
+ from RunSpec for executor runtime use.
+ type: object
+ group:
+ description: Group is the group this action belongs to, if applicable.
+ maxLength: 256
+ type: string
+ inputUri:
+ description: InputURI is the path to the input data for this action
+ minLength: 1
+ type: string
+ interruptible:
+ description: Interruptible is the run-scoped interruptibility override
+ projected from RunSpec.
+ type: boolean
+ parentActionName:
+ description: ParentActionName is the optional name of the parent action
+ maxLength: 30
+ minLength: 1
+ type: string
+ project:
+ description: Project this action belongs to
+ maxLength: 63
+ minLength: 1
+ type: string
+ runName:
+ description: RunName is the name of the run this action belongs to
+ maxLength: 30
+ minLength: 1
+ type: string
+ runOutputBase:
+ description: RunOutputBase is the base path where this action should
+ write its output
+ minLength: 1
+ type: string
+ shortName:
+ description: ShortName is the human-readable display name for this
+ task
+ maxLength: 63
+ type: string
+ taskTemplate:
+ description: TaskTemplate is the proto-serialized core.TaskTemplate
+ stored inline in etcd
+ format: byte
+ type: string
+ taskType:
+ description: TaskType identifies which plugin handles this task (e.g.
+ "container", "spark", "ray")
+ maxLength: 63
+ minLength: 1
+ type: string
+ required:
+ - actionName
+ - domain
+ - inputUri
+ - project
+ - runName
+ - runOutputBase
+ - taskTemplate
+ - taskType
+ type: object
+ status:
+ description: status defines the observed state of TaskAction
+ properties:
+ attempts:
+ description: Attempts is the latest observed action attempt number,
+ starting from 1.
+ format: int32
+ type: integer
+ cacheStatus:
+ description: CacheStatus is the latest observed cache lookup result
+ for this action.
+ format: int32
+ type: integer
+ conditions:
+ description: |-
+ conditions represent the current state of the TaskAction resource.
+ Each condition has a unique type and reflects the status of a specific aspect of the resource.
+
+ Standard condition types include:
+ - "Available": the resource is fully functional
+ - "Progressing": the resource is being created or updated
+ - "Degraded": the resource failed to reach or maintain its desired state
+
+ The status of each condition is one of True, False, or Unknown.
+ items:
+ description: Condition contains details for one aspect of the current
+ state of this API Resource.
+ properties:
+ lastTransitionTime:
+ description: |-
+ lastTransitionTime is the last time the condition transitioned from one status to another.
+ This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
+ format: date-time
+ type: string
+ message:
+ description: |-
+ message is a human readable message indicating details about the transition.
+ This may be an empty string.
+ maxLength: 32768
+ type: string
+ observedGeneration:
+ description: |-
+ observedGeneration represents the .metadata.generation that the condition was set based upon.
+ For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+ with respect to the current state of the instance.
+ format: int64
+ minimum: 0
+ type: integer
+ reason:
+ description: |-
+ reason contains a programmatic identifier indicating the reason for the condition's last transition.
+ Producers of specific condition types may define expected values and meanings for this field,
+ and whether the values are considered a guaranteed API.
+ The value should be a CamelCase string.
+ This field may not be empty.
+ maxLength: 1024
+ minLength: 1
+ pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+ type: string
+ status:
+ description: status of the condition, one of True, False, Unknown.
+ enum:
+ - "True"
+ - "False"
+ - Unknown
+ type: string
+ type:
+ description: type of condition in CamelCase or in foo.example.com/CamelCase.
+ maxLength: 316
+ pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+ type: string
+ required:
+ - lastTransitionTime
+ - message
+ - reason
+ - status
+ - type
+ type: object
+ type: array
+ x-kubernetes-list-map-keys:
+ - type
+ x-kubernetes-list-type: map
+ phaseHistory:
+ description: |-
+ PhaseHistory is an append-only log of phase transitions. Unlike conditions
+ (which are updated in-place by type), this preserves the full timeline:
+ Queued → Initializing → Executing → Succeeded/Failed, each with a timestamp.
+ items:
+ description: PhaseTransition records a phase change with its timestamp.
+ properties:
+ message:
+ description: Message is an optional human-readable message about
+ the transition.
+ type: string
+ occurredAt:
+ description: OccurredAt is when this phase transition happened.
+ format: date-time
+ type: string
+ phase:
+ description: Phase is the phase that was entered (e.g. "Queued",
+ "Initializing", "Executing", "Succeeded", "Failed").
+ type: string
+ required:
+ - occurredAt
+ - phase
+ type: object
+ type: array
+ pluginPhase:
+ description: PluginPhase is a human-readable representation of the
+ plugin's current phase.
+ type: string
+ pluginPhaseVersion:
+ description: PluginPhaseVersion is the version of the current plugin
+ phase.
+ format: int32
+ type: integer
+ pluginState:
+ description: PluginState is the Gob-encoded plugin state from the
+ last reconciliation round.
+ format: byte
+ type: string
+ pluginStateVersion:
+ description: PluginStateVersion tracks the version of the plugin state
+ schema for compatibility.
+ type: integer
+ stateJson:
+ description: StateJSON is the JSON serialized NodeStatus that was
+ last sent to the State Service
+ type: string
+ type: object
+ required:
+ - spec
+ type: object
+ served: true
+ storage: true
+ subresources:
+ status: {}
+---
+apiVersion: v1
+data:
+ config.yml: |-
+ health:
+ storagedriver:
+ enabled: true
+ interval: 10s
+ threshold: 3
+ http:
+ addr: :5000
+ debug:
+ addr: :5001
+ prometheus:
+ enabled: false
+ path: /metrics
+ headers:
+ X-Content-Type-Options:
+ - nosniff
+ log:
+ fields:
+ service: registry
+ storage:
+ cache:
+ blobdescriptor: inmemory
+ version: 0.1
+kind: ConfigMap
+metadata:
+ labels:
+ app: docker-registry
+ chart: docker-registry-2.2.2
+ heritage: Helm
+ release: flyte-devbox
+ name: docker-registry-config
+ namespace: flyte
+---
+apiVersion: v1
+data:
+ haSharedSecret: Zmx5dGVzYW5kYm94c2VjcmV0
+ proxyPassword: ""
+ proxyUsername: ""
+kind: Secret
+metadata:
+ labels:
+ app: docker-registry
+ chart: docker-registry-2.2.2
+ heritage: Helm
+ release: flyte-devbox
+ name: docker-registry-secret
+ namespace: flyte
+type: Opaque
+---
+apiVersion: v1
+data:
+ access-key: cnVzdGZz
+ secret-key: cnVzdGZzc3RvcmFnZQ==
+kind: Secret
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: rustfs
+ namespace: flyte
+type: Opaque
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: flyte-devbox-local
+ namespace: flyte
+subsets:
+- addresses:
+ - ip: '%{HOST_GATEWAY_IP}%'
+ ports:
+ - name: http
+ port: 8090
+ protocol: TCP
+ - name: webhook
+ port: 9443
+ protocol: TCP
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+ labels:
+ app.kubernetes.io/name: embedded-postgresql
+ name: postgresql
+ namespace: flyte
+subsets:
+- addresses:
+ - ip: '%{NODE_IP}%'
+ ports:
+ - name: tcp-postgresql
+ port: 5432
+---
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app: docker-registry
+ chart: docker-registry-2.2.2
+ heritage: Helm
+ release: flyte-devbox
+ name: docker-registry
+ namespace: flyte
+spec:
+ ports:
+ - name: http-5000
+ nodePort: 30000
+ port: 5000
+ protocol: TCP
+ targetPort: 5000
+ selector:
+ app: docker-registry
+ release: flyte-devbox
+ type: NodePort
+---
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: flyte-console
+ namespace: flyte
+spec:
+ ports:
+ - name: http
+ port: 80
+ protocol: TCP
+ targetPort: http
+ selector:
+ app.kubernetes.io/component: console
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: flyte-devbox
+---
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: flyte-devbox-local
+ namespace: flyte
+spec:
+ clusterIP: None
+ ports:
+ - name: http
+ port: 8090
+ protocol: TCP
+ - name: webhook
+ port: 9443
+ protocol: TCP
+---
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/name: embedded-postgresql
+ name: postgresql
+ namespace: flyte
+spec:
+ ports:
+ - name: tcp-postgresql
+ nodePort: 30001
+ port: 5432
+ targetPort: 5432
+ type: NodePort
+---
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: rustfs
+ namespace: flyte
+spec:
+ ports:
+ - name: rustfs-api
+ nodePort: 30002
+ port: 9000
+ targetPort: rustfs-api
+ selector:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: rustfs
+ type: NodePort
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: flyte-devbox-rustfs-storage
+ namespace: flyte
+spec:
+ accessModes:
+ - ReadWriteOnce
+ capacity:
+ storage: 1Gi
+ hostPath:
+ path: /var/lib/flyte/storage/rustfs
+ storageClassName: manual
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: flyte-devbox-rustfs-storage
+ namespace: flyte
+spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ storageClassName: manual
+ volumeName: flyte-devbox-rustfs-storage
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ labels:
+ app: docker-registry
+ chart: docker-registry-2.2.2
+ heritage: Helm
+ release: flyte-devbox
+ name: docker-registry
+ namespace: flyte
+spec:
+ minReadySeconds: 5
+ replicas: 1
+ selector:
+ matchLabels:
+ app: docker-registry
+ release: flyte-devbox
+ template:
+ metadata:
+ annotations:
+ checksum/config: 6b06ba712a995bdcc044732aa5e5b9d56ce24061c6ad5c2b7f35508a00c7863b
+ checksum/secret: 8f6b14d94a45833195baa1b863e43296aa157e65ef50c983e672c34648d36c50
+ labels:
+ app: docker-registry
+ release: flyte-devbox
+ spec:
+ containers:
+ - command:
+ - /bin/registry
+ - serve
+ - /etc/docker/registry/config.yml
+ env:
+ - name: REGISTRY_HTTP_SECRET
+ valueFrom:
+ secretKeyRef:
+ key: haSharedSecret
+ name: docker-registry-secret
+ - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
+ value: /var/lib/registry
+ image: registry:sandbox
+ imagePullPolicy: Never
+ livenessProbe:
+ httpGet:
+ path: /
+ port: 5000
+ name: docker-registry
+ ports:
+ - containerPort: 5000
+ readinessProbe:
+ httpGet:
+ path: /
+ port: 5000
+ resources: {}
+ volumeMounts:
+ - mountPath: /etc/docker/registry
+ name: docker-registry-config
+ - mountPath: /var/lib/registry/
+ name: data
+ securityContext:
+ fsGroup: 1000
+ runAsUser: 1000
+ volumes:
+ - configMap:
+ name: docker-registry-config
+ name: docker-registry-config
+ - emptyDir: {}
+ name: data
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: flyte-console
+ namespace: flyte
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app.kubernetes.io/component: console
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: flyte-devbox
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/component: console
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: flyte-devbox
+ spec:
+ containers:
+ - image: docker.io/unionai-oss/flyteconsole-v2:sandbox
+ imagePullPolicy: Never
+ livenessProbe:
+ httpGet:
+ path: /v2
+ port: http
+ initialDelaySeconds: 5
+ periodSeconds: 30
+ name: console
+ ports:
+ - containerPort: 8080
+ name: http
+ protocol: TCP
+ readinessProbe:
+ httpGet:
+ path: /v2
+ port: http
+ initialDelaySeconds: 5
+ periodSeconds: 10
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: rustfs
+ namespace: flyte
+spec:
+ selector:
+ matchLabels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: rustfs
+ strategy:
+ type: Recreate
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/name: rustfs
+ spec:
+ containers:
+ - env:
+ - name: RUSTFS_ADDRESS
+ value: 0.0.0.0:9000
+ - name: RUSTFS_VOLUMES
+ value: /data
+ - name: RUSTFS_ACCESS_KEY
+ valueFrom:
+ secretKeyRef:
+ key: access-key
+ name: rustfs
+ - name: RUSTFS_SECRET_KEY
+ valueFrom:
+ secretKeyRef:
+ key: secret-key
+ name: rustfs
+ image: rustfs/rustfs:sandbox
+ imagePullPolicy: Never
+ livenessProbe:
+ failureThreshold: 5
+ initialDelaySeconds: 10
+ periodSeconds: 10
+ tcpSocket:
+ port: rustfs-api
+ name: rustfs
+ ports:
+ - containerPort: 9000
+ name: rustfs-api
+ protocol: TCP
+ readinessProbe:
+ failureThreshold: 5
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ tcpSocket:
+ port: rustfs-api
+ securityContext:
+ runAsNonRoot: true
+ runAsUser: 10001
+ volumeMounts:
+ - mountPath: /data
+ name: data
+ initContainers:
+ - command:
+ - /bin/sh
+ - -ec
+ - |
+ chown -R 10001:10001 /data
+ mkdir -p /data/flyte-data
+ chown 10001:10001 /data/flyte-data
+ image: busybox:latest
+ imagePullPolicy: IfNotPresent
+ name: volume-permissions
+ securityContext:
+ runAsUser: 0
+ volumeMounts:
+ - mountPath: /data
+ name: data
+ securityContext:
+ fsGroup: 10001
+ volumes:
+ - name: data
+ persistentVolumeClaim:
+ claimName: flyte-devbox-rustfs-storage
+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: traefik
+ namespace: kube-system
+spec:
+ valuesContent: |
+ service:
+ type: NodePort
+ ports:
+ web:
+ nodePort: 30080
+ transport:
+ respondingTimeouts:
+ readTimeout: 0
+ websecure:
+ expose: false
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: flyte-console
+ namespace: flyte
+spec:
+ rules:
+ - http:
+ paths:
+ - backend:
+ service:
+ name: flyte-console
+ port:
+ number: 80
+ path: /v2
+ pathType: Prefix
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+ labels:
+ app.kubernetes.io/instance: flyte-devbox
+ app.kubernetes.io/managed-by: Helm
+ app.kubernetes.io/name: flyte-devbox
+ app.kubernetes.io/version: 1.16.1
+ helm.sh/chart: flyte-devbox-0.1.0
+ name: flyte-devbox-api
+ namespace: flyte
+spec:
+ rules:
+ - http:
+ paths:
+ - backend:
+ service:
+ name: flyte-devbox-local
+ port:
+ number: 8090
+ path: /healthz
+ pathType: Exact
+ - backend:
+ service:
+ name: flyte-devbox-local
+ port:
+ number: 8090
+ path: /readyz
+ pathType: Exact
+ - backend:
+ service:
+ name: flyte-devbox-local
+ port:
+ number: 8090
+ path: /flyteidl2.
+ pathType: Prefix
diff --git a/docker/devbox-bundled/nvidia-device-plugin.yaml b/docker/devbox-bundled/nvidia-device-plugin.yaml
new file mode 100644
index 00000000000..f0bbfcdb01c
--- /dev/null
+++ b/docker/devbox-bundled/nvidia-device-plugin.yaml
@@ -0,0 +1,45 @@
+apiVersion: node.k8s.io/v1
+kind: RuntimeClass
+metadata:
+ name: nvidia
+handler: nvidia
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: nvidia-device-plugin-daemonset
+ namespace: kube-system
+spec:
+ selector:
+ matchLabels:
+ name: nvidia-device-plugin-ds
+ updateStrategy:
+ type: RollingUpdate
+ template:
+ metadata:
+ labels:
+ name: nvidia-device-plugin-ds
+ spec:
+ tolerations:
+ - key: nvidia.com/gpu
+ operator: Exists
+ effect: NoSchedule
+ priorityClassName: system-node-critical
+ runtimeClassName: nvidia
+ containers:
+ - name: nvidia-device-plugin-ctr
+ image: nvcr.io/nvidia/k8s-device-plugin:v0.17.0
+ env:
+ - name: FAIL_ON_INIT_ERROR
+ value: "false"
+ securityContext:
+ allowPrivilegeEscalation: false
+ capabilities:
+ drop: ["ALL"]
+ volumeMounts:
+ - name: device-plugin
+ mountPath: /var/lib/kubelet/device-plugins
+ volumes:
+ - name: device-plugin
+ hostPath:
+ path: /var/lib/kubelet/device-plugins
diff --git a/docs/docker-image-workflow.md b/docs/docker-image-workflow.md
new file mode 100644
index 00000000000..d1cd423897c
--- /dev/null
+++ b/docs/docker-image-workflow.md
@@ -0,0 +1,310 @@
+# Docker Image Build Workflow
+
+This document explains how the Docker CI image is built and used across different scenarios.
+
+## Workflow Diagrams
+
+### Scenario 1: Regular PR (No Dockerfile Changes)
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│ Developer creates PR (no gen.Dockerfile changes) │
+└─────────────────┬───────────────────────────────────────────┘
+ │
+ ▼
+┌─────────────────────────────────────────────────────────────┐
+│ check-generate workflow triggers │
+│ • Checks if gen.Dockerfile modified: NO │
+│ • Uses image: ghcr.io/flyteorg/flyte/ci:v2 │
+└─────────────────┬───────────────────────────────────────────┘
+ │
+ ▼
+┌─────────────────────────────────────────────────────────────┐
+│ Pulls existing v2 image and runs checks │
+│ ✓ Fast: No image build needed │
+└─────────────────────────────────────────────────────────────┘
+```
+
+### Scenario 2: PR with Dockerfile Changes
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│ Developer creates PR (modifies gen.Dockerfile) │
+└─────────────────┬───────────────────────────────────────────┘
+ │
+ ├──────────────────────┬─────────────────────┐
+ ▼ ▼ ▼
+ ┌────────────────┐ ┌─────────────────┐ ┌──────────────┐
+ │ build-ci-image │ │ check-generate │ │ Other checks │
+ │ workflow │ │ workflow │ │ │
+ └────────┬───────┘ └────────┬────────┘ └──────┬───────┘
+ │ │ │
+ ▼ ▼ ▼
+ ┌────────────────┐ ┌─────────────────┐ ┌──────────────┐
+ │ Builds image │ │ Detects Docker- │ │ Uses PR │
+ │ with tag: │ │ file modified │ │ image │
+ │ pr-123 │ │ │ │ │
+ └────────┬───────┘ └────────┬────────┘ └──────────────┘
+ │ │
+ │ ▼
+ │ ┌─────────────────┐
+ │ │ Waits for image │
+ │ │ to be available │
+ │ │ (polls for ~10m)│
+ │ └────────┬────────┘
+ │ │
+ ▼ ▼
+ ┌────────────────────────────────────┐
+ │ Pushes to ghcr.io │
+ │ ghcr.io/flyteorg/flyte/ci:pr-123 │
+ └────────┬───────────────────────────┘
+ │
+ ▼
+ ┌────────────────┐
+ │ Comments on PR │
+ │ with image tag │
+ └────────────────┘
+ │
+ ▼
+ ┌────────────────────────────────────┐
+ │ All workflows use pr-123 image │
+ │ Developer can test same image │
+ └────────────────────────────────────┘
+```
+
+### Scenario 3: Merged to v2 Branch
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│ PR merged to v2 branch │
+└─────────────────┬───────────────────────────────────────────┘
+ │
+ ▼
+┌─────────────────────────────────────────────────────────────┐
+│ build-ci-image workflow triggers on push │
+└─────────────────┬───────────────────────────────────────────┘
+ │
+ ▼
+┌─────────────────────────────────────────────────────────────┐
+│ Builds and pushes with tags: │
+│ • ghcr.io/flyteorg/flyte/ci:v2 │
+│ • ghcr.io/flyteorg/flyte/ci:v2-sha-abc123 │
+└─────────────────┬───────────────────────────────────────────┘
+ │
+ ▼
+┌─────────────────────────────────────────────────────────────┐
+│ Future PRs use updated v2 image │
+└─────────────────────────────────────────────────────────────┘
+```
+
+## Image Tag Strategy
+
+| Context | Image Tag | When Created |
+|--------------------|----------------------------------------|---------------------------------|
+| Regular PR | `v2` or `latest` | Uses existing branch image |
+| PR with Dockerfile | `pr-123` | Built when PR is created/updated|
+| v2 branch push | `v2` | Built on every push to v2 |
+| main branch push | `latest` | Built on every push to master |
+| Any branch push | `{branch}-sha-{commit}` | Built on push (with commit SHA) |
+
+## Workflow Files
+
+### Primary Workflows
+
+1. **`.github/workflows/build-ci-image.yml`**
+ - Builds Docker image
+ - Runs on: PR with Dockerfile changes, push to main/v2
+ - Publishes to GHCR with appropriate tags
+ - Comments on PR with image information
+
+2. **`.github/workflows/check-generate.yml`**
+ - Validates generated files
+ - Automatically detects if Dockerfile was modified
+ - Uses PR-specific image if available, otherwise uses v2
+
+3. **`.github/workflows/regenerate-on-comment.yml`**
+ - Regenerates files via `/regen` comment
+ - Uses PR-specific image if available
+
+## Developer Experience
+
+### For Regular Development (No Docker Changes)
+
+```bash
+# Just use the standard v2 image
+make docker-pull
+make gen
+```
+
+### For Docker Image Updates
+
+#### Option 1: Local Development (Fastest - Recommended)
+
+```bash
+# Modify gen.Dockerfile
+vim gen.Dockerfile
+
+# Build and test locally
+make docker-dev
+
+# Iterate quickly
+vim gen.Dockerfile
+make docker-build # Uses cache, faster rebuilds
+make gen
+
+# When it works, push to PR
+git commit -am "Update Python to 3.13"
+git push
+```
+
+**Benefits:**
+- ⚡ Fast iteration (seconds to minutes)
+- 🔄 No network dependency
+- 🧪 Test before pushing
+- 💰 No CI minutes wasted
+
+#### Option 2: PR-Based Testing
+
+```bash
+# 1. Modify gen.Dockerfile
+vim gen.Dockerfile
+
+# 2. Create PR
+git checkout -b update-python
+git commit -am "Update Python to 3.13"
+git push origin update-python
+# Create PR on GitHub
+
+# 3. Wait for build (~5-10 minutes)
+# Bot will comment: "🐳 Docker CI Image Built"
+# Image: ghcr.io/flyteorg/flyte/ci:pr-456
+
+# 4. Test locally with YOUR image
+make docker-pull gen DOCKER_CI_IMAGE=ghcr.io/flyteorg/flyte/ci:pr-456
+
+# 5. CI automatically uses the same image!
+# No need to merge before testing
+```
+
+## Benefits of This Approach
+
+1. **Test Before Merge**: You can fully test Docker image changes before merging
+2. **True Parity**: Local testing uses the exact same image as CI
+3. **Fast Iteration**: No need to merge to test, iterate in the PR
+4. **Automatic Detection**: Workflows automatically detect which image to use
+5. **Clean Fallback**: If Docker isn't modified, uses the stable v2 image
+6. **Transparent**: Bot comments show exactly what image is being used
+
+## Implementation Details
+
+### Image Detection Logic
+
+Workflows check if `gen.Dockerfile` or `.github/workflows/build-ci-image.yml` were modified:
+
+```bash
+git diff --name-only origin/$BASE_BRANCH...HEAD | \
+ grep -E '^(Dockerfile\.ci|\.github/workflows/build-ci-image\.yml)$'
+```
+
+If modified:
+- Use `ghcr.io/flyteorg/flyte/ci:pr-{NUMBER}`
+- Wait for image to be available (with timeout)
+
+If not modified:
+- Use `ghcr.io/flyteorg/flyte/ci:v2`
+- Proceed immediately
+
+### Wait Strategy
+
+For PRs with Dockerfile changes, workflows intelligently wait for the build:
+
+1. **Check for build workflow**: Queries GitHub API for `build-ci-image.yml` runs
+2. **Find matching run**: Looks for run with same commit SHA
+3. **Wait for completion**: Polls every 20 seconds for up to 20 minutes
+4. **Verify success**: Ensures build succeeded before proceeding
+5. **Pull fresh image**: Downloads the newly built image
+
+**Benefits:**
+- Always waits for the actual build to complete (not just image existence)
+- Works correctly on subsequent pushes to the same PR
+- Provides clear feedback on build status
+- Fails fast if build fails
+
+This ensures tests always run with the freshly built image, even when pushing multiple commits to a PR.
+
+## Build Performance
+
+The Docker image is optimized for fast builds using several techniques:
+
+### Multi-Stage Build Strategy
+
+The Dockerfile uses parallel multi-stage builds to download tools simultaneously:
+
+```
+┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐
+│ Go Stage │ │ Node Stage │ │Python Stage │ │ Buf Stage │
+│ (parallel) │ │ (parallel) │ │ (parallel) │ │ (parallel) │
+└──────┬──────┘ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘
+ │ │ │ │
+ └────────────────┴────────────────┴────────────────┘
+ │
+ ┌──────▼──────┐
+ │Final Image │
+ │ (assembly) │
+ └─────────────┘
+```
+
+**Benefits:**
+- 4x parallelization of downloads
+- Leverages official Docker images (pre-built, cached)
+- Faster than sequential downloads
+
+### Caching Strategy
+
+The build uses a multi-layer caching approach:
+
+1. **Registry cache** (primary): Stored in GHCR, fastest to pull
+2. **GitHub Actions cache** (secondary): Fallback for layers
+3. **BuildKit inline cache**: Metadata in image layers
+4. **Cache mounts**: For package managers (apt, go mod, cargo)
+
+**Cache hierarchy:**
+```
+1. Try buildcache tag (dedicated cache image)
+2. Try current PR tag (if exists)
+3. Try v2 tag (stable baseline)
+4. Try GHA cache
+5. Build from scratch
+```
+
+### Performance Improvements
+
+| Optimization | Time Saved |
+|--------------|------------|
+| Multi-stage parallel builds | ~5-8 min |
+| Official image copying vs downloads | ~2-3 min |
+| Registry cache (vs no cache) | ~10-12 min |
+| Cache mounts for packages | ~1-2 min |
+| **Total potential savings** | **15-20 min** |
+
+**Build times:**
+- Cold build (no cache): ~15 min
+- Warm build (full cache): ~2-3 min
+- Incremental build (partial cache): ~5-8 min
+
+### Cache Mounts
+
+The Dockerfile uses BuildKit cache mounts for package managers:
+
+```dockerfile
+# APT packages cached
+RUN --mount=type=cache,target=/var/cache/apt
+
+# Go modules cached
+RUN --mount=type=cache,target=/root/go/pkg/mod
+
+# Cargo packages cached
+RUN --mount=type=cache,target=/root/.cargo/registry
+```
+
+These persist across builds, dramatically speeding up package installation.
diff --git a/events/cmd/main.go b/events/cmd/main.go
new file mode 100644
index 00000000000..cabbf351a66
--- /dev/null
+++ b/events/cmd/main.go
@@ -0,0 +1,27 @@
+package main
+
+import (
+ "context"
+ "os"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/app"
+ "github.com/flyteorg/flyte/v2/events"
+ eventsconfig "github.com/flyteorg/flyte/v2/events/config"
+)
+
+func main() {
+ a := &app.App{
+ Name: "events-service",
+ Short: "Events Service for Flyte",
+ Setup: func(ctx context.Context, sc *app.SetupContext) error {
+ cfg := eventsconfig.GetConfig()
+ sc.Host = cfg.Server.Host
+ sc.Port = cfg.Server.Port
+
+ return events.Setup(ctx, sc)
+ },
+ }
+ if err := a.Run(); err != nil {
+ os.Exit(1)
+ }
+}
diff --git a/events/config/config.go b/events/config/config.go
new file mode 100644
index 00000000000..7d53e544de3
--- /dev/null
+++ b/events/config/config.go
@@ -0,0 +1,37 @@
+package config
+
+import "github.com/flyteorg/flyte/v2/flytestdlib/config"
+
+const configSectionKey = "events"
+
+//go:generate pflags Config --default-var=defaultConfig
+
+var defaultConfig = &Config{
+ Server: ServerConfig{
+ Port: 8092,
+ Host: "0.0.0.0",
+ },
+ RunServiceURL: "http://localhost:8090",
+}
+
+var configSection = config.MustRegisterSection(configSectionKey, defaultConfig)
+
+// Config holds configuration for the Events service.
+type Config struct {
+ // HTTP server configuration.
+ Server ServerConfig `json:"server"`
+
+ // RunServiceURL is the base URL for the internal run service.
+ RunServiceURL string `json:"runServiceUrl" pflag:",Base URL of the internal run service"`
+}
+
+// ServerConfig holds HTTP server configuration.
+type ServerConfig struct {
+ Port int `json:"port" pflag:",Port to bind the HTTP server"`
+ Host string `json:"host" pflag:",Host to bind the HTTP server"`
+}
+
+// GetConfig returns the parsed events configuration.
+func GetConfig() *Config {
+ return configSection.GetConfig().(*Config)
+}
diff --git a/events/service/events_proxy_service.go b/events/service/events_proxy_service.go
new file mode 100644
index 00000000000..93f9b4c21ce
--- /dev/null
+++ b/events/service/events_proxy_service.go
@@ -0,0 +1,48 @@
+package service
+
+import (
+ "context"
+ "fmt"
+
+ "connectrpc.com/connect"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow/workflowconnect"
+)
+
+// EventsProxyService implements the EventsProxyService gRPC API.
+type EventsProxyService struct {
+ runClient workflowconnect.InternalRunServiceClient
+}
+
+func NewEventsProxyService(runClient workflowconnect.InternalRunServiceClient) *EventsProxyService {
+ return &EventsProxyService{runClient: runClient}
+}
+
+// Record accepts action events and forwards them synchronously to the run service.
+func (s *EventsProxyService) Record(ctx context.Context, req *connect.Request[workflow.RecordRequest]) (*connect.Response[workflow.RecordResponse], error) {
+ if s.runClient == nil {
+ return nil, connect.NewError(connect.CodeFailedPrecondition, fmt.Errorf("run client is not initialized"))
+ }
+ if err := req.Msg.Validate(); err != nil {
+ logger.Errorf(ctx, "invalid EventsProxyService.Record request: %v", err)
+ return nil, connect.NewError(connect.CodeInvalidArgument, err)
+ }
+ if len(req.Msg.GetEvents()) == 0 {
+ return connect.NewResponse(&workflow.RecordResponse{}), nil
+ }
+
+ recordEventReq := &workflow.RecordActionEventsRequest{Events: req.Msg.GetEvents()}
+ recordActionResp, err := s.runClient.RecordActionEvents(ctx, connect.NewRequest(recordEventReq))
+ if err != nil {
+ logger.Warnf(ctx, "failed to forward action events to run service: %v", err)
+ return nil, connect.NewError(connect.CodeInternal, err)
+ }
+ status := recordActionResp.Msg.GetStatus()
+ if status != nil && status.Code != 0 {
+ logger.Warnf(ctx, "run service returned non-ok status for action events: code=%d, msg=%s", status.Code, status.Message)
+ return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("run service failed: %s", status.Message))
+ }
+
+ return connect.NewResponse(&workflow.RecordResponse{}), nil
+}
diff --git a/events/setup.go b/events/setup.go
new file mode 100644
index 00000000000..9750d69d961
--- /dev/null
+++ b/events/setup.go
@@ -0,0 +1,30 @@
+package events
+
+import (
+ "context"
+ "net/http"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/app"
+ "github.com/flyteorg/flyte/v2/events/config"
+ "github.com/flyteorg/flyte/v2/events/service"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow/workflowconnect"
+)
+
+// Setup registers the EventsProxyService handler.
+func Setup(ctx context.Context, sc *app.SetupContext) error {
+ cfg := config.GetConfig()
+
+ runServiceURL := cfg.RunServiceURL
+ if sc.BaseURL != "" {
+ runServiceURL = sc.BaseURL
+ }
+ runClient := workflowconnect.NewInternalRunServiceClient(http.DefaultClient, runServiceURL)
+
+ eventsSvc := service.NewEventsProxyService(runClient)
+ path, handler := workflowconnect.NewEventsProxyServiceHandler(eventsSvc)
+ sc.Mux.Handle(path, handler)
+ logger.Infof(ctx, "Mounted EventsProxyService at %s", path)
+
+ return nil
+}
diff --git a/executor/.devcontainer/devcontainer.json b/executor/.devcontainer/devcontainer.json
new file mode 100644
index 00000000000..a3ab7541cb6
--- /dev/null
+++ b/executor/.devcontainer/devcontainer.json
@@ -0,0 +1,25 @@
+{
+ "name": "Kubebuilder DevContainer",
+ "image": "golang:1.24",
+ "features": {
+ "ghcr.io/devcontainers/features/docker-in-docker:2": {},
+ "ghcr.io/devcontainers/features/git:1": {}
+ },
+
+ "runArgs": ["--network=host"],
+
+ "customizations": {
+ "vscode": {
+ "settings": {
+ "terminal.integrated.shell.linux": "/bin/bash"
+ },
+ "extensions": [
+ "ms-kubernetes-tools.vscode-kubernetes-tools",
+ "ms-azuretools.vscode-docker"
+ ]
+ }
+ },
+
+ "onCreateCommand": "bash .devcontainer/post-install.sh"
+}
+
diff --git a/executor/.devcontainer/post-install.sh b/executor/.devcontainer/post-install.sh
new file mode 100644
index 00000000000..265c43ee8d3
--- /dev/null
+++ b/executor/.devcontainer/post-install.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+set -x
+
+curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64
+chmod +x ./kind
+mv ./kind /usr/local/bin/kind
+
+curl -L -o kubebuilder https://go.kubebuilder.io/dl/latest/linux/amd64
+chmod +x kubebuilder
+mv kubebuilder /usr/local/bin/
+
+KUBECTL_VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt)
+curl -LO "https://dl.k8s.io/release/$KUBECTL_VERSION/bin/linux/amd64/kubectl"
+chmod +x kubectl
+mv kubectl /usr/local/bin/kubectl
+
+docker network create -d=bridge --subnet=172.19.0.0/24 kind
+
+kind version
+kubebuilder version
+docker --version
+go version
+kubectl version --client
diff --git a/executor/.dockerignore b/executor/.dockerignore
new file mode 100644
index 00000000000..315b541c982
--- /dev/null
+++ b/executor/.dockerignore
@@ -0,0 +1,13 @@
+# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
+# Ignore everything by default and re-include only needed files
+**
+
+# Re-include Go source files (but not *_test.go)
+!**/*.go
+**/*_test.go
+
+# Re-include Go module files
+!go.mod
+!go.sum
+
+bin/
\ No newline at end of file
diff --git a/executor/.github/workflows/lint.yml b/executor/.github/workflows/lint.yml
new file mode 100644
index 00000000000..187b5b9d787
--- /dev/null
+++ b/executor/.github/workflows/lint.yml
@@ -0,0 +1,23 @@
+name: Lint
+
+on:
+ push:
+ pull_request:
+
+jobs:
+ lint:
+ name: Run on Ubuntu
+ runs-on: ubuntu-latest
+ steps:
+ - name: Clone the code
+ uses: actions/checkout@v4
+
+ - name: Setup Go
+ uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+
+ - name: Run linter
+ uses: golangci/golangci-lint-action@v8
+ with:
+ version: v2.4.0
diff --git a/executor/.github/workflows/test-e2e.yml b/executor/.github/workflows/test-e2e.yml
new file mode 100644
index 00000000000..68fd1ed5562
--- /dev/null
+++ b/executor/.github/workflows/test-e2e.yml
@@ -0,0 +1,32 @@
+name: E2E Tests
+
+on:
+ push:
+ pull_request:
+
+jobs:
+ test-e2e:
+ name: Run on Ubuntu
+ runs-on: ubuntu-latest
+ steps:
+ - name: Clone the code
+ uses: actions/checkout@v4
+
+ - name: Setup Go
+ uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+
+ - name: Install the latest version of kind
+ run: |
+ curl -Lo ./kind https://kind.sigs.k8s.io/dl/latest/kind-linux-amd64
+ chmod +x ./kind
+ sudo mv ./kind /usr/local/bin/kind
+
+ - name: Verify kind installation
+ run: kind version
+
+ - name: Running Test e2e
+ run: |
+ go mod tidy
+ make test-e2e
diff --git a/executor/.github/workflows/test.yml b/executor/.github/workflows/test.yml
new file mode 100644
index 00000000000..fc2e80d304d
--- /dev/null
+++ b/executor/.github/workflows/test.yml
@@ -0,0 +1,23 @@
+name: Tests
+
+on:
+ push:
+ pull_request:
+
+jobs:
+ test:
+ name: Run on Ubuntu
+ runs-on: ubuntu-latest
+ steps:
+ - name: Clone the code
+ uses: actions/checkout@v4
+
+ - name: Setup Go
+ uses: actions/setup-go@v5
+ with:
+ go-version-file: go.mod
+
+ - name: Running Tests
+ run: |
+ go mod tidy
+ make test
diff --git a/executor/.gitignore b/executor/.gitignore
new file mode 100644
index 00000000000..ada68ff086c
--- /dev/null
+++ b/executor/.gitignore
@@ -0,0 +1,27 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+bin/*
+Dockerfile.cross
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Go workspace file
+go.work
+
+# Kubernetes Generated files - skip generated files, except for vendored files
+!vendor/**/zz_generated.*
+
+# editor and IDE paraphernalia
+.idea
+.vscode
+*.swp
+*.swo
+*~
diff --git a/executor/.golangci.yml b/executor/.golangci.yml
new file mode 100644
index 00000000000..e5b21b0f11c
--- /dev/null
+++ b/executor/.golangci.yml
@@ -0,0 +1,52 @@
+version: "2"
+run:
+ allow-parallel-runners: true
+linters:
+ default: none
+ enable:
+ - copyloopvar
+ - dupl
+ - errcheck
+ - ginkgolinter
+ - goconst
+ - gocyclo
+ - govet
+ - ineffassign
+ - lll
+ - misspell
+ - nakedret
+ - prealloc
+ - revive
+ - staticcheck
+ - unconvert
+ - unparam
+ - unused
+ settings:
+ revive:
+ rules:
+ - name: comment-spacings
+ - name: import-shadowing
+ exclusions:
+ generated: lax
+ rules:
+ - linters:
+ - lll
+ path: api/*
+ - linters:
+ - dupl
+ - lll
+ path: internal/*
+ paths:
+ - third_party$
+ - builtin$
+ - examples$
+formatters:
+ enable:
+ - gofmt
+ - goimports
+ exclusions:
+ generated: lax
+ paths:
+ - third_party$
+ - builtin$
+ - examples$
diff --git a/executor/DEVELOPMENT.md b/executor/DEVELOPMENT.md
new file mode 100644
index 00000000000..b7187212b1a
--- /dev/null
+++ b/executor/DEVELOPMENT.md
@@ -0,0 +1,154 @@
+# Executor Development Guide
+
+This guide provides steps on how to develop and iterates changes.
+
+## Prerequisites
+- go version v1.24.0+
+- docker version 17.03+.
+- kubectl version v1.11.3+.
+
+### Setup on kind
+
+We recommend using kind to create a Kubernetes cluster for local development.
+
+### Use go v1.24
+
+Currently, flyte-v2 use go v1.24 for development.
+
+```sh
+go install golang.org/dl/go1.24.0@latest
+go1.24.0 download
+export GOROOT=$(go1.24.0 env GOROOT)
+export PATH="$GOROOT/bin:$PATH"
+```
+
+### Clean up local binaries
+
+To keep consistent code generation and testing result, it is recommended to remove outdated binaries installed by the Makefile.
+
+```sh
+make clean
+```
+
+## Local development on kind cluster
+
+Following steps requires you to switch your working directory to the `executor/`.
+
+```sh
+cd executor
+```
+
+### Run executor inside the cluster
+
+1. Create a kind cluster
+
+```sh
+kind create cluster --image=kindest/node:v1.34.0 --name flytev2
+```
+
+2. Build the image
+
+```sh
+IMG=flyteorg/executor:nightly make docker-build
+```
+
+3. Load image into kind cluster
+
+```sh
+kind load docker-image flyteorg/executor:nightly --name flytev2
+```
+
+4. Install CRD into the cluster
+
+```sh
+make install
+```
+
+5. Deploy the Manager to the cluster with the image specified by `IMG`:
+
+```sh
+make deploy IMG=flyteorg/executor:nightly
+```
+
+6. Clean up
+
+```sh
+kind delete cluster --name flytev2
+```
+
+### Run executor outside the cluster
+
+1. Create a kind cluster
+
+```sh
+kind create cluster --image=kindest/node:v1.34.0 --name flytev2
+```
+
+2. Install CRD into the cluster
+
+```sh
+make install
+```
+
+3. Compile the source code and run
+
+```sh
+# Compile the source code
+make build
+# Run the executor
+./bin/manager
+```
+
+## Tests
+
+### Manual test
+
+You can apply the samples from the config/sample using the following command:
+
+```sh
+kubectl apply -k config/samples/
+```
+
+You can see the `TaskAction` CRD created:
+
+```sh
+❯ kubectl get taskactions
+NAME RUN ACTION STATUS AGE
+taskaction-sample sample-run sample-task Completed 59m
+```
+
+Or use `-o wide` to view more details:
+
+```sh
+❯ kubectl get taskactions -o wide
+NAME RUN ACTION STATUS AGE PROGRESSING SUCCEEDED FAILED
+taskaction-sample sample-run sample-task Completed 59m False True
+```
+
+## Modify CRD
+
+### Quick Steps
+
+1. Modify the types file: Edit `api/v1/taskaction_types.go`
+ - Add/modify fields in `TaskActionSpec` or `TaskActionStatus`
+ - Add/modify printcolumns (the `// +kubebuilder:printcolumn` comments above the `TaskAction` struct)
+ - Add validation rules using `// +kubebuilder:validation:` markers
+
+2. Generate CRD manifests and DeepCopy code
+
+```sh
+make manifests generate
+```
+
+This runs two commands:
+- `make manifests`: Updates YAML manifests:
+ - `config/crd/bases/flyte.org_taskactions.yaml` (CRD with schema, validation, printcolumns)
+ - `config/rbac/role.yaml` (RBAC permissions)
+- `make generate`: Updates Go code:
+ - `api/v1/zz_generated.deepcopy.go` (DeepCopy methods required by Kubernetes)
+
+3. Update CRD in the cluster
+
+```sh
+make install
+```
diff --git a/executor/Dockerfile b/executor/Dockerfile
new file mode 100644
index 00000000000..7f70c66dc2e
--- /dev/null
+++ b/executor/Dockerfile
@@ -0,0 +1,32 @@
+# Build the manager binary
+FROM golang:1.24 AS builder
+ARG TARGETOS
+ARG TARGETARCH
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy only the needed source files from monorepo
+COPY executor/ executor/
+COPY gen/ gen/
+
+# Build the executor binary
+# the GOARCH has no default value to allow the binary to be built according to the host where the command
+# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
+# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
+# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager executor/cmd/main.go
+
+# Use distroless as minimal base image to package the manager binary
+# Refer to https://github.com/GoogleContainerTools/distroless for more details
+FROM gcr.io/distroless/static:nonroot
+WORKDIR /
+COPY --from=builder /workspace/manager .
+USER 65532:65532
+
+ENTRYPOINT ["/manager"]
diff --git a/executor/Makefile b/executor/Makefile
new file mode 100644
index 00000000000..64b2d484294
--- /dev/null
+++ b/executor/Makefile
@@ -0,0 +1,245 @@
+# Image URL to use all building/pushing image targets
+IMG ?= controller:latest
+
+# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
+ifeq (,$(shell go env GOBIN))
+GOBIN=$(shell go env GOPATH)/bin
+else
+GOBIN=$(shell go env GOBIN)
+endif
+
+# CONTAINER_TOOL defines the container tool to be used for building images.
+# Be aware that the target commands are only tested with Docker which is
+# scaffolded by default. However, you might want to replace it to use other
+# tools. (i.e. podman)
+CONTAINER_TOOL ?= docker
+
+# Setting SHELL to bash allows bash commands to be executed by recipes.
+# Options are set to exit when a recipe line exits non-zero or a piped command fails.
+SHELL = /usr/bin/env bash -o pipefail
+.SHELLFLAGS = -ec
+
+.PHONY: all
+all: build
+
+##@ General
+
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk command is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
+
+.PHONY: help
+help: ## Display this help.
+ @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+##@ Development
+
+.PHONY: manifests
+manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
+ $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+ cp config/crd/bases/*.yaml ../charts/flyte-binary/templates/crds/
+
+.PHONY: generate
+generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+ $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
+
+.PHONY: fmt
+fmt: ## Run go fmt against code.
+ go fmt ./...
+
+.PHONY: vet
+vet: ## Run go vet against code.
+ go vet ./...
+
+.PHONY: test
+test: manifests generate fmt vet setup-envtest ## Run tests.
+ KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
+
+# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
+# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
+# CertManager is installed by default; skip with:
+# - CERT_MANAGER_INSTALL_SKIP=true
+KIND_CLUSTER ?= executor-test-e2e
+
+.PHONY: setup-test-e2e
+setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist
+ @command -v $(KIND) >/dev/null 2>&1 || { \
+ echo "Kind is not installed. Please install Kind manually."; \
+ exit 1; \
+ }
+ @case "$$($(KIND) get clusters)" in \
+ *"$(KIND_CLUSTER)"*) \
+ echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \
+ *) \
+ echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \
+ $(KIND) create cluster --name $(KIND_CLUSTER) ;; \
+ esac
+
+.PHONY: test-e2e
+test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind.
+ KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v
+ $(MAKE) cleanup-test-e2e
+
+.PHONY: cleanup-test-e2e
+cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests
+ @$(KIND) delete cluster --name $(KIND_CLUSTER)
+
+.PHONY: lint
+lint: golangci-lint ## Run golangci-lint linter
+ $(GOLANGCI_LINT) run
+
+.PHONY: lint-fix
+lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes
+ $(GOLANGCI_LINT) run --fix
+
+.PHONY: lint-config
+lint-config: golangci-lint ## Verify golangci-lint linter configuration
+ $(GOLANGCI_LINT) config verify
+
+##@ Build
+
+.PHONY: build
+build: manifests generate fmt vet ## Build manager binary.
+ go build -o bin/manager cmd/main.go
+
+.PHONY: run
+run: manifests generate fmt vet ## Run a controller from your host.
+ go run ./cmd/main.go
+
+# If you wish to build the manager image targeting other platforms you can use the --platform flag.
+# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
+# More info: https://docs.docker.com/develop/develop-images/build_enhancements/
+.PHONY: docker-build
+docker-build: ## Build docker image with the manager.
+ $(CONTAINER_TOOL) build -t ${IMG} -f Dockerfile ..
+
+.PHONY: docker-push
+docker-push: ## Push docker image with the manager.
+ $(CONTAINER_TOOL) push ${IMG}
+
+# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
+# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
+# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/
+# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/
+# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail)
+# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option.
+PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le
+.PHONY: docker-buildx
+docker-buildx: ## Build and push docker image for the manager for cross-platform support
+ # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile
+ sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
+ - $(CONTAINER_TOOL) buildx create --name executor-builder
+ $(CONTAINER_TOOL) buildx use executor-builder
+ - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross .
+ - $(CONTAINER_TOOL) buildx rm executor-builder
+ rm Dockerfile.cross
+
+.PHONY: build-installer
+build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment.
+ mkdir -p dist
+ cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+ $(KUSTOMIZE) build config/default > dist/install.yaml
+
+##@ Deployment
+
+ifndef ignore-not-found
+ ignore-not-found = false
+endif
+
+.PHONY: install
+install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+ @out="$$( $(KUSTOMIZE) build config/crd 2>/dev/null || true )"; \
+ if [ -n "$$out" ]; then echo "$$out" | $(KUBECTL) apply -f -; else echo "No CRDs to install; skipping."; fi
+
+.PHONY: uninstall
+uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+ @out="$$( $(KUSTOMIZE) build config/crd 2>/dev/null || true )"; \
+ if [ -n "$$out" ]; then echo "$$out" | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -; else echo "No CRDs to delete; skipping."; fi
+
+.PHONY: deploy
+deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+ cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+ $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
+
+.PHONY: undeploy
+undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+ $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
+
+##@ Dependencies
+
+## Location to install dependencies to
+LOCALBIN ?= $(shell pwd)/bin
+$(LOCALBIN):
+ mkdir -p $(LOCALBIN)
+
+.PHONY: clean
+clean: ## Cleanup local binaries such as controller-gen and kustomize
+ rm -rf $(LOCALBIN)
+
+## Tool Binaries
+KUBECTL ?= kubectl
+KIND ?= kind
+KUSTOMIZE ?= $(LOCALBIN)/kustomize
+CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
+ENVTEST ?= $(LOCALBIN)/setup-envtest
+GOLANGCI_LINT = $(LOCALBIN)/golangci-lint
+
+## Tool Versions
+KUSTOMIZE_VERSION ?= v5.7.1
+CONTROLLER_TOOLS_VERSION ?= v0.19.0
+#ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20)
+ENVTEST_VERSION ?= $(shell go list -m -f "{{ .Version }}" sigs.k8s.io/controller-runtime | awk -F'[v.]' '{printf "release-%d.%d", $$2, $$3}')
+#ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31)
+ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk -F'[v.]' '{printf "1.%d", $$3}')
+GOLANGCI_LINT_VERSION ?= v2.4.0
+
+.PHONY: kustomize
+kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
+$(KUSTOMIZE): $(LOCALBIN)
+ $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION))
+
+.PHONY: controller-gen
+controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
+$(CONTROLLER_GEN): $(LOCALBIN)
+ $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION))
+
+.PHONY: setup-envtest
+setup-envtest: envtest ## Download the binaries required for ENVTEST in the local bin directory.
+ @echo "Setting up envtest binaries for Kubernetes version $(ENVTEST_K8S_VERSION)..."
+ @$(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path || { \
+ echo "Error: Failed to set up envtest binaries for version $(ENVTEST_K8S_VERSION)."; \
+ exit 1; \
+ }
+
+.PHONY: envtest
+envtest: $(ENVTEST) ## Download setup-envtest locally if necessary.
+$(ENVTEST): $(LOCALBIN)
+ $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION))
+
+.PHONY: golangci-lint
+golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
+$(GOLANGCI_LINT): $(LOCALBIN)
+ $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION))
+
+# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
+# $1 - target path with name of binary
+# $2 - package url which can be installed
+# $3 - specific version of package
+define go-install-tool
+@[ -f "$(1)-$(3)" ] && [ "$$(readlink -- "$(1)" 2>/dev/null)" = "$(1)-$(3)" ] || { \
+set -e; \
+package=$(2)@$(3) ;\
+echo "Downloading $${package}" ;\
+rm -f $(1) ;\
+GOBIN=$(LOCALBIN) go install $${package} ;\
+mv $(1) $(1)-$(3) ;\
+} ;\
+ln -sf $$(realpath $(1)-$(3)) $(1)
+endef
diff --git a/executor/PROJECT b/executor/PROJECT
new file mode 100644
index 00000000000..f3b14c92e8e
--- /dev/null
+++ b/executor/PROJECT
@@ -0,0 +1,21 @@
+# Code generated by tool. DO NOT EDIT.
+# This file is used to track the info used to scaffold your project
+# and allow the plugins properly work.
+# More info: https://book.kubebuilder.io/reference/project-config.html
+cliVersion: 4.9.0
+domain: flyte.org
+layout:
+- go.kubebuilder.io/v4
+projectName: executor
+repo: github.com/flyteorg/flyte/v2/executor
+resources:
+- api:
+ crdVersion: v1
+ namespaced: true
+ controller: true
+ domain: flyte.org
+ group: flyte.org
+ kind: TaskAction
+ path: github.com/flyteorg/flyte/v2/executor/api/v1
+ version: v1
+version: "3"
diff --git a/executor/README.md b/executor/README.md
new file mode 100644
index 00000000000..583215a077b
--- /dev/null
+++ b/executor/README.md
@@ -0,0 +1,150 @@
+# Executor
+
+The Executor is a Kubernetes operator that manages the execution of Flyte tasks in the data plane. Built using the Kubernetes Operator pattern with `controller-runtime`, it watches and reconciles `TaskAction` custom resources to orchestrate task execution.
+
+## Description
+
+The Executor controller is responsible for:
+- **Monitoring TaskAction CRs**: Watches for TaskAction resources created in the Kubernetes cluster
+- **Task Execution**: Executes tasks based on the specifications defined in TaskAction resources
+- **State Management**: Reports task execution state to the State Service
+- **Lifecycle Management**: Manages the full lifecycle of task execution from queued to completed/failed states
+
+The executor uses conditions to track task progress:
+- `Progressing`: Indicates active execution (reasons: Queued, Initializing, Executing)
+- `Succeeded`: Task completed successfully
+- `Failed`: Task execution failed
+
+
+## Getting Started
+
+### Prerequisites
+- go version v1.24.0+
+- docker version 17.03+.
+- kubectl version v1.11.3+.
+- Access to a Kubernetes v1.11.3+ cluster.
+
+### To Deploy on the cluster
+**Build and push your image to the location specified by `IMG`:**
+
+```sh
+make docker-build docker-push IMG=/executor:tag
+```
+
+**NOTE:** This image ought to be published in the personal registry you specified.
+And it is required to have access to pull the image from the working environment.
+Make sure you have the proper permission to the registry if the above commands don’t work.
+
+**Install the CRDs into the cluster:**
+
+```sh
+make install
+```
+
+**Deploy the Manager to the cluster with the image specified by `IMG`:**
+
+```sh
+make deploy IMG=/executor:tag
+```
+
+> **NOTE**: If you encounter RBAC errors, you may need to grant yourself cluster-admin
+privileges or be logged in as admin.
+
+**Create instances of your solution**
+You can apply the samples (examples) from the config/sample:
+
+```sh
+kubectl apply -k config/samples/
+```
+
+>**NOTE**: Ensure that the samples has default values to test it out.
+
+### To Uninstall
+**Delete the instances (CRs) from the cluster:**
+
+```sh
+kubectl delete -k config/samples/
+```
+
+**Delete the APIs(CRDs) from the cluster:**
+
+```sh
+make uninstall
+```
+
+**UnDeploy the controller from the cluster:**
+
+```sh
+make undeploy
+```
+
+## Project Distribution
+
+Following the options to release and provide this solution to the users.
+
+### By providing a bundle with all YAML files
+
+1. Build the installer for the image built and published in the registry:
+
+```sh
+make build-installer IMG=/executor:tag
+```
+
+**NOTE:** The makefile target mentioned above generates an 'install.yaml'
+file in the dist directory. This file contains all the resources built
+with Kustomize, which are necessary to install this project without its
+dependencies.
+
+2. Using the installer
+
+Users can just run 'kubectl apply -f ' to install
+the project, i.e.:
+
+```sh
+kubectl apply -f https://raw.githubusercontent.com//executor//dist/install.yaml
+```
+
+### By providing a Helm Chart
+
+1. Build the chart using the optional helm plugin
+
+```sh
+kubebuilder edit --plugins=helm/v1-alpha
+```
+
+2. See that a chart was generated under 'dist/chart', and users
+can obtain this solution from there.
+
+**NOTE:** If you change the project, you need to update the Helm Chart
+using the same command above to sync the latest changes. Furthermore,
+if you create webhooks, you need to use the above command with
+the '--force' flag and manually ensure that any custom configuration
+previously added to 'dist/chart/values.yaml' or 'dist/chart/manager/manager.yaml'
+is manually re-applied afterwards.
+
+
+## Contributing
+
+Please read our [CONTRIBUTING](../CONTRIBUTING.md) guide before making a pull request. Refer to our
+[DEVELOPMENT.md](DEVELOPMENT.md) to build and run tests for executor locally.
+
+**NOTE:** Run `make help` for more information on all potential `make` targets
+
+More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html)
+
+## License
+
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
diff --git a/executor/api/v1/groupversion_info.go b/executor/api/v1/groupversion_info.go
new file mode 100644
index 00000000000..ad3d9014381
--- /dev/null
+++ b/executor/api/v1/groupversion_info.go
@@ -0,0 +1,36 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package v1 contains API Schema definitions for the flyte.org v1 API group.
+// +kubebuilder:object:generate=true
+// +groupName=flyte.org
+package v1
+
+import (
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ "sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+var (
+ // GroupVersion is group version used to register these objects.
+ GroupVersion = schema.GroupVersion{Group: "flyte.org", Version: "v1"}
+
+ // SchemeBuilder is used to add go types to the GroupVersionKind scheme.
+ SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+
+ // AddToScheme adds the types in this group-version to the given scheme.
+ AddToScheme = SchemeBuilder.AddToScheme
+)
diff --git a/executor/api/v1/taskaction_types.go b/executor/api/v1/taskaction_types.go
new file mode 100644
index 00000000000..d9c2d7f0b7f
--- /dev/null
+++ b/executor/api/v1/taskaction_types.go
@@ -0,0 +1,310 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+import (
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow"
+)
+
+// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.
+
+type (
+ TaskActionConditionType string
+ TaskActionConditionReason string
+)
+
+// Condition type constants
+// Following Kubernetes API conventions:
+// - Condition types describe the current observed state
+// - Use Reason field to track sub-states (like Queued, Initializing, Executing)
+const (
+ // ConditionTypeProgressing indicates whether the TaskAction is actively progressing.
+ // This is True when the TaskAction is queued, initializing, or executing.
+ // This is False when the TaskAction has completed or failed.
+ ConditionTypeProgressing TaskActionConditionType = "Progressing"
+
+ // ConditionTypeSucceeded indicates whether the TaskAction has completed successfully.
+ // This is a terminal condition. Once True, the TaskAction will not be reconciled further.
+ ConditionTypeSucceeded TaskActionConditionType = "Succeeded"
+
+ // ConditionTypeFailed indicates whether the TaskAction has failed.
+ // This is a terminal condition. Once True, the TaskAction will not be reconciled further.
+ ConditionTypeFailed TaskActionConditionType = "Failed"
+)
+
+// Condition reason constants
+// Reasons explain why a condition has a particular status.
+// These are used in the Reason field of conditions to provide detailed sub-state information.
+const (
+ // ConditionReasonQueued indicates the TaskAction is queued and waiting for resources
+ ConditionReasonQueued TaskActionConditionReason = "Queued"
+
+ // ConditionReasonInitializing indicates the TaskAction is being initialized
+ ConditionReasonInitializing TaskActionConditionReason = "Initializing"
+
+ // ConditionReasonExecuting indicates the TaskAction is actively executing
+ ConditionReasonExecuting TaskActionConditionReason = "Executing"
+
+ // ConditionReasonCompleted indicates the TaskAction has completed successfully
+ ConditionReasonCompleted TaskActionConditionReason = "Completed"
+
+ // ConditionReasonRetryableFailure indicates the TaskAction experienced a retryable failure
+ ConditionReasonRetryableFailure TaskActionConditionReason = "RetryableFailure"
+
+ // ConditionReasonPermanentFailure indicates the TaskAction experienced a permanent failure
+ ConditionReasonPermanentFailure TaskActionConditionReason = "PermanentFailure"
+
+ // ConditionReasonAborted indicates the TaskAction was aborted
+ ConditionReasonAborted TaskActionConditionReason = "Aborted"
+
+ // ConditionReasonPluginNotFound indicates no plugin was found for the task type
+ ConditionReasonPluginNotFound TaskActionConditionReason = "PluginNotFound"
+
+ // ConditionReasonInvalidSpec indicates the TaskAction spec is missing required fields
+ ConditionReasonInvalidSpec TaskActionConditionReason = "InvalidSpec"
+)
+
+// TaskActionSpec defines the desired state of TaskAction
+type TaskActionSpec struct {
+ // RunName is the name of the run this action belongs to
+ // +kubebuilder:validation:Required
+ // +kubebuilder:validation:MinLength=1
+ // +kubebuilder:validation:MaxLength=30
+ RunName string `json:"runName"`
+
+ // Project this action belongs to
+ // +kubebuilder:validation:Required
+ // +kubebuilder:validation:MinLength=1
+ // +kubebuilder:validation:MaxLength=63
+ Project string `json:"project"`
+
+ // Domain this action belongs to
+ // +kubebuilder:validation:Required
+ // +kubebuilder:validation:MinLength=1
+ // +kubebuilder:validation:MaxLength=63
+ Domain string `json:"domain"`
+
+ // ActionName is the unique name of this action within the run
+ // +kubebuilder:validation:Required
+ // +kubebuilder:validation:MinLength=1
+ // +kubebuilder:validation:MaxLength=30
+ ActionName string `json:"actionName"`
+
+ // ParentActionName is the optional name of the parent action
+ // +optional
+ // +kubebuilder:validation:MinLength=1
+ // +kubebuilder:validation:MaxLength=30
+ ParentActionName *string `json:"parentActionName,omitempty"`
+
+ // InputURI is the path to the input data for this action
+ // +kubebuilder:validation:Required
+ // +kubebuilder:validation:MinLength=1
+ InputURI string `json:"inputUri"`
+
+ // RunOutputBase is the base path where this action should write its output
+ // +kubebuilder:validation:Required
+ // +kubebuilder:validation:MinLength=1
+ RunOutputBase string `json:"runOutputBase"`
+
+ // CacheKey enables cache lookup/writeback for this task action when set.
+ // This is propagated from workflow.TaskAction.cache_key.
+ // +optional
+ // +kubebuilder:validation:MaxLength=256
+ CacheKey string `json:"cacheKey,omitempty"`
+
+ // TaskType identifies which plugin handles this task (e.g. "container", "spark", "ray")
+ // +kubebuilder:validation:Required
+ // +kubebuilder:validation:MinLength=1
+ // +kubebuilder:validation:MaxLength=63
+ TaskType string `json:"taskType"`
+
+ // ShortName is the human-readable display name for this task
+ // +optional
+ // +kubebuilder:validation:MaxLength=63
+ ShortName string `json:"shortName,omitempty"`
+
+ // TaskTemplate is the proto-serialized core.TaskTemplate stored inline in etcd
+ // +kubebuilder:validation:Required
+ TaskTemplate []byte `json:"taskTemplate"`
+
+ // EnvVars are run-scoped environment variables projected from RunSpec for executor runtime use.
+ // +optional
+ EnvVars map[string]string `json:"envVars,omitempty"`
+
+ // Interruptible is the run-scoped interruptibility override projected from RunSpec.
+ // +optional
+ Interruptible *bool `json:"interruptible,omitempty"`
+
+ // Group is the group this action belongs to, if applicable.
+ // +optional
+ // +kubebuilder:validation:MaxLength=256
+ Group string `json:"group,omitempty"`
+}
+
+func (in *TaskActionSpec) GetActionSpec() (*workflow.ActionSpec, error) {
+ // Build ActionSpec from structured fields
+ spec := &workflow.ActionSpec{
+ ActionId: &common.ActionIdentifier{
+ Run: &common.RunIdentifier{
+ Project: in.Project,
+ Domain: in.Domain,
+ Name: in.RunName,
+ },
+ Name: in.ActionName,
+ },
+ ParentActionName: in.ParentActionName,
+ InputUri: in.InputURI,
+ RunOutputBase: in.RunOutputBase,
+ Group: in.Group,
+ }
+
+ return spec, nil
+}
+
+func (in *TaskActionSpec) SetActionSpec(spec *workflow.ActionSpec) error {
+ // Populate structured fields from ActionSpec
+ if spec.ActionId != nil {
+ if spec.ActionId.Run != nil {
+ in.Project = spec.ActionId.Run.Project
+ in.Domain = spec.ActionId.Run.Domain
+ in.RunName = spec.ActionId.Run.Name
+ }
+ in.ActionName = spec.ActionId.Name
+ }
+ in.ParentActionName = spec.ParentActionName
+ in.InputURI = spec.InputUri
+ in.RunOutputBase = spec.RunOutputBase
+ in.Group = spec.Group
+
+ return nil
+}
+
+// PhaseTransition records a phase change with its timestamp.
+type PhaseTransition struct {
+ // Phase is the phase that was entered (e.g. "Queued", "Initializing", "Executing", "Succeeded", "Failed").
+ Phase string `json:"phase"`
+
+ // OccurredAt is when this phase transition happened.
+ OccurredAt metav1.Time `json:"occurredAt"`
+
+ // Message is an optional human-readable message about the transition.
+ // +optional
+ Message string `json:"message,omitempty"`
+}
+
+// TaskActionStatus defines the observed state of TaskAction.
+type TaskActionStatus struct {
+ // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
+ // Important: Run "make" to regenerate code after modifying this file
+
+ // For Kubernetes API conventions, see:
+ // https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties
+
+ // StateJSON is the JSON serialized NodeStatus that was last sent to the State Service
+ // +optional
+ StateJSON string `json:"stateJson,omitempty"`
+
+ // PluginState is the Gob-encoded plugin state from the last reconciliation round.
+ // +optional
+ PluginState []byte `json:"pluginState,omitempty"`
+
+ // PluginStateVersion tracks the version of the plugin state schema for compatibility.
+ // +optional
+ PluginStateVersion uint8 `json:"pluginStateVersion,omitempty"`
+
+ // PluginPhase is a human-readable representation of the plugin's current phase.
+ // +optional
+ PluginPhase string `json:"pluginPhase,omitempty"`
+
+ // PluginPhaseVersion is the version of the current plugin phase.
+ // +optional
+ PluginPhaseVersion uint32 `json:"pluginPhaseVersion,omitempty"`
+
+ // Attempts is the latest observed action attempt number, starting from 1.
+ // +optional
+ Attempts uint32 `json:"attempts,omitempty"`
+
+ // CacheStatus is the latest observed cache lookup result for this action.
+ // +optional
+ CacheStatus core.CatalogCacheStatus `json:"cacheStatus,omitempty"`
+
+ // conditions represent the current state of the TaskAction resource.
+ // Each condition has a unique type and reflects the status of a specific aspect of the resource.
+ //
+ // Standard condition types include:
+ // - "Available": the resource is fully functional
+ // - "Progressing": the resource is being created or updated
+ // - "Degraded": the resource failed to reach or maintain its desired state
+ //
+ // The status of each condition is one of True, False, or Unknown.
+ // +listType=map
+ // +listMapKey=type
+ // +optional
+ Conditions []metav1.Condition `json:"conditions,omitempty"`
+
+ // PhaseHistory is an append-only log of phase transitions. Unlike conditions
+ // (which are updated in-place by type), this preserves the full timeline:
+ // Queued → Initializing → Executing → Succeeded/Failed, each with a timestamp.
+ // +optional
+ PhaseHistory []PhaseTransition `json:"phaseHistory,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="Run",type="string",JSONPath=".spec.runName"
+// +kubebuilder:printcolumn:name="Action",type="string",JSONPath=".spec.actionName"
+// +kubebuilder:printcolumn:name="TaskType",type="string",JSONPath=".spec.taskType"
+// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=".status.conditions[?(@.type=='Progressing')].reason"
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+// +kubebuilder:printcolumn:name="Progressing",type="string",JSONPath=".status.conditions[?(@.type=='Progressing')].status",priority=1
+// +kubebuilder:printcolumn:name="Succeeded",type="string",JSONPath=".status.conditions[?(@.type=='Succeeded')].status",priority=1
+// +kubebuilder:printcolumn:name="Failed",type="string",JSONPath=".status.conditions[?(@.type=='Failed')].status",priority=1
+
+// TaskAction is the Schema for the taskactions API
+type TaskAction struct {
+ metav1.TypeMeta `json:",inline"`
+
+ // metadata is a standard object metadata
+ // +optional
+ metav1.ObjectMeta `json:"metadata,omitempty,omitzero"`
+
+ // spec defines the desired state of TaskAction
+ // +required
+ Spec TaskActionSpec `json:"spec"`
+
+ // status defines the observed state of TaskAction
+ // +optional
+ Status TaskActionStatus `json:"status,omitempty,omitzero"`
+}
+
+// +kubebuilder:object:root=true
+
+// TaskActionList contains a list of TaskAction
+type TaskActionList struct {
+ metav1.TypeMeta `json:",inline"`
+ metav1.ListMeta `json:"metadata,omitempty"`
+ Items []TaskAction `json:"items"`
+}
+
+func init() {
+ SchemeBuilder.Register(&TaskAction{}, &TaskActionList{})
+}
diff --git a/executor/api/v1/zz_generated.deepcopy.go b/executor/api/v1/zz_generated.deepcopy.go
new file mode 100644
index 00000000000..6e22f81f68f
--- /dev/null
+++ b/executor/api/v1/zz_generated.deepcopy.go
@@ -0,0 +1,172 @@
+//go:build !ignore_autogenerated
+
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by controller-gen. DO NOT EDIT.
+
+package v1
+
+import (
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ runtime "k8s.io/apimachinery/pkg/runtime"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PhaseTransition) DeepCopyInto(out *PhaseTransition) {
+ *out = *in
+ in.OccurredAt.DeepCopyInto(&out.OccurredAt)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PhaseTransition.
+func (in *PhaseTransition) DeepCopy() *PhaseTransition {
+ if in == nil {
+ return nil
+ }
+ out := new(PhaseTransition)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TaskAction) DeepCopyInto(out *TaskAction) {
+ *out = *in
+ out.TypeMeta = in.TypeMeta
+ in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+ in.Spec.DeepCopyInto(&out.Spec)
+ in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskAction.
+func (in *TaskAction) DeepCopy() *TaskAction {
+ if in == nil {
+ return nil
+ }
+ out := new(TaskAction)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *TaskAction) DeepCopyObject() runtime.Object {
+ if c := in.DeepCopy(); c != nil {
+ return c
+ }
+ return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TaskActionList) DeepCopyInto(out *TaskActionList) {
+ *out = *in
+ out.TypeMeta = in.TypeMeta
+ in.ListMeta.DeepCopyInto(&out.ListMeta)
+ if in.Items != nil {
+ in, out := &in.Items, &out.Items
+ *out = make([]TaskAction, len(*in))
+ for i := range *in {
+ (*in)[i].DeepCopyInto(&(*out)[i])
+ }
+ }
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskActionList.
+func (in *TaskActionList) DeepCopy() *TaskActionList {
+ if in == nil {
+ return nil
+ }
+ out := new(TaskActionList)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *TaskActionList) DeepCopyObject() runtime.Object {
+ if c := in.DeepCopy(); c != nil {
+ return c
+ }
+ return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TaskActionSpec) DeepCopyInto(out *TaskActionSpec) {
+ *out = *in
+ if in.ParentActionName != nil {
+ in, out := &in.ParentActionName, &out.ParentActionName
+ *out = new(string)
+ **out = **in
+ }
+ if in.TaskTemplate != nil {
+ in, out := &in.TaskTemplate, &out.TaskTemplate
+ *out = make([]byte, len(*in))
+ copy(*out, *in)
+ }
+ if in.EnvVars != nil {
+ in, out := &in.EnvVars, &out.EnvVars
+ *out = make(map[string]string, len(*in))
+ for key, val := range *in {
+ (*out)[key] = val
+ }
+ }
+ if in.Interruptible != nil {
+ in, out := &in.Interruptible, &out.Interruptible
+ *out = new(bool)
+ **out = **in
+ }
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskActionSpec.
+func (in *TaskActionSpec) DeepCopy() *TaskActionSpec {
+ if in == nil {
+ return nil
+ }
+ out := new(TaskActionSpec)
+ in.DeepCopyInto(out)
+ return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TaskActionStatus) DeepCopyInto(out *TaskActionStatus) {
+ *out = *in
+ if in.PluginState != nil {
+ in, out := &in.PluginState, &out.PluginState
+ *out = make([]byte, len(*in))
+ copy(*out, *in)
+ }
+ if in.Conditions != nil {
+ in, out := &in.Conditions, &out.Conditions
+ *out = make([]metav1.Condition, len(*in))
+ for i := range *in {
+ (*in)[i].DeepCopyInto(&(*out)[i])
+ }
+ }
+ if in.PhaseHistory != nil {
+ in, out := &in.PhaseHistory, &out.PhaseHistory
+ *out = make([]PhaseTransition, len(*in))
+ for i := range *in {
+ (*in)[i].DeepCopyInto(&(*out)[i])
+ }
+ }
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaskActionStatus.
+func (in *TaskActionStatus) DeepCopy() *TaskActionStatus {
+ if in == nil {
+ return nil
+ }
+ out := new(TaskActionStatus)
+ in.DeepCopyInto(out)
+ return out
+}
diff --git a/executor/cmd/main.go b/executor/cmd/main.go
new file mode 100644
index 00000000000..5fdc9718036
--- /dev/null
+++ b/executor/cmd/main.go
@@ -0,0 +1,32 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/app"
+ "github.com/flyteorg/flyte/v2/executor"
+ ctrl "sigs.k8s.io/controller-runtime"
+)
+
+func main() {
+ a := &app.App{
+ Name: "executor",
+ Short: "Executor controller manager for Flyte TaskActions",
+ Setup: func(ctx context.Context, sc *app.SetupContext) error {
+ // Executor doesn't serve HTTP. Use 0 to avoid binding an app HTTP port.
+ sc.Port = 0
+ sc.K8sConfig = ctrl.GetConfigOrDie()
+
+ if err := executor.Setup(ctx, sc); err != nil {
+ return fmt.Errorf("executor setup failed: %w", err)
+ }
+ return nil
+ },
+ }
+
+ if err := a.Run(); err != nil {
+ os.Exit(1)
+ }
+}
diff --git a/executor/config/crd/bases/flyte.org_taskactions.yaml b/executor/config/crd/bases/flyte.org_taskactions.yaml
new file mode 100644
index 00000000000..969d28e00a8
--- /dev/null
+++ b/executor/config/crd/bases/flyte.org_taskactions.yaml
@@ -0,0 +1,287 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+ annotations:
+ controller-gen.kubebuilder.io/version: v0.19.0
+ name: taskactions.flyte.org
+spec:
+ group: flyte.org
+ names:
+ kind: TaskAction
+ listKind: TaskActionList
+ plural: taskactions
+ singular: taskaction
+ scope: Namespaced
+ versions:
+ - additionalPrinterColumns:
+ - jsonPath: .spec.runName
+ name: Run
+ type: string
+ - jsonPath: .spec.actionName
+ name: Action
+ type: string
+ - jsonPath: .spec.taskType
+ name: TaskType
+ type: string
+ - jsonPath: .status.conditions[?(@.type=='Progressing')].reason
+ name: Status
+ type: string
+ - jsonPath: .metadata.creationTimestamp
+ name: Age
+ type: date
+ - jsonPath: .status.conditions[?(@.type=='Progressing')].status
+ name: Progressing
+ priority: 1
+ type: string
+ - jsonPath: .status.conditions[?(@.type=='Succeeded')].status
+ name: Succeeded
+ priority: 1
+ type: string
+ - jsonPath: .status.conditions[?(@.type=='Failed')].status
+ name: Failed
+ priority: 1
+ type: string
+ name: v1
+ schema:
+ openAPIV3Schema:
+ description: TaskAction is the Schema for the taskactions API
+ properties:
+ apiVersion:
+ description: |-
+ APIVersion defines the versioned schema of this representation of an object.
+ Servers should convert recognized schemas to the latest internal value, and
+ may reject unrecognized values.
+ More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+ type: string
+ kind:
+ description: |-
+ Kind is a string value representing the REST resource this object represents.
+ Servers may infer this from the endpoint the client submits requests to.
+ Cannot be updated.
+ In CamelCase.
+ More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+ type: string
+ metadata:
+ type: object
+ spec:
+ description: spec defines the desired state of TaskAction
+ properties:
+ actionName:
+ description: ActionName is the unique name of this action within the
+ run
+ maxLength: 30
+ minLength: 1
+ type: string
+ cacheKey:
+ description: |-
+ CacheKey enables cache lookup/writeback for this task action when set.
+ This is propagated from workflow.TaskAction.cache_key.
+ maxLength: 256
+ type: string
+ domain:
+ description: Domain this action belongs to
+ maxLength: 63
+ minLength: 1
+ type: string
+ envVars:
+ additionalProperties:
+ type: string
+ description: EnvVars are run-scoped environment variables projected
+ from RunSpec for executor runtime use.
+ type: object
+ group:
+ description: Group is the group this action belongs to, if applicable.
+ maxLength: 256
+ type: string
+ inputUri:
+ description: InputURI is the path to the input data for this action
+ minLength: 1
+ type: string
+ interruptible:
+ description: Interruptible is the run-scoped interruptibility override
+ projected from RunSpec.
+ type: boolean
+ parentActionName:
+ description: ParentActionName is the optional name of the parent action
+ maxLength: 30
+ minLength: 1
+ type: string
+ project:
+ description: Project this action belongs to
+ maxLength: 63
+ minLength: 1
+ type: string
+ runName:
+ description: RunName is the name of the run this action belongs to
+ maxLength: 30
+ minLength: 1
+ type: string
+ runOutputBase:
+ description: RunOutputBase is the base path where this action should
+ write its output
+ minLength: 1
+ type: string
+ shortName:
+ description: ShortName is the human-readable display name for this
+ task
+ maxLength: 63
+ type: string
+ taskTemplate:
+ description: TaskTemplate is the proto-serialized core.TaskTemplate
+ stored inline in etcd
+ format: byte
+ type: string
+ taskType:
+ description: TaskType identifies which plugin handles this task (e.g.
+ "container", "spark", "ray")
+ maxLength: 63
+ minLength: 1
+ type: string
+ required:
+ - actionName
+ - domain
+ - inputUri
+ - project
+ - runName
+ - runOutputBase
+ - taskTemplate
+ - taskType
+ type: object
+ status:
+ description: status defines the observed state of TaskAction
+ properties:
+ attempts:
+ description: Attempts is the latest observed action attempt number,
+ starting from 1.
+ format: int32
+ type: integer
+ cacheStatus:
+ description: CacheStatus is the latest observed cache lookup result
+ for this action.
+ format: int32
+ type: integer
+ conditions:
+ description: |-
+ conditions represent the current state of the TaskAction resource.
+ Each condition has a unique type and reflects the status of a specific aspect of the resource.
+
+ Standard condition types include:
+ - "Available": the resource is fully functional
+ - "Progressing": the resource is being created or updated
+ - "Degraded": the resource failed to reach or maintain its desired state
+
+ The status of each condition is one of True, False, or Unknown.
+ items:
+ description: Condition contains details for one aspect of the current
+ state of this API Resource.
+ properties:
+ lastTransitionTime:
+ description: |-
+ lastTransitionTime is the last time the condition transitioned from one status to another.
+ This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
+ format: date-time
+ type: string
+ message:
+ description: |-
+ message is a human readable message indicating details about the transition.
+ This may be an empty string.
+ maxLength: 32768
+ type: string
+ observedGeneration:
+ description: |-
+ observedGeneration represents the .metadata.generation that the condition was set based upon.
+ For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+ with respect to the current state of the instance.
+ format: int64
+ minimum: 0
+ type: integer
+ reason:
+ description: |-
+ reason contains a programmatic identifier indicating the reason for the condition's last transition.
+ Producers of specific condition types may define expected values and meanings for this field,
+ and whether the values are considered a guaranteed API.
+ The value should be a CamelCase string.
+ This field may not be empty.
+ maxLength: 1024
+ minLength: 1
+ pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+ type: string
+ status:
+ description: status of the condition, one of True, False, Unknown.
+ enum:
+ - "True"
+ - "False"
+ - Unknown
+ type: string
+ type:
+ description: type of condition in CamelCase or in foo.example.com/CamelCase.
+ maxLength: 316
+ pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+ type: string
+ required:
+ - lastTransitionTime
+ - message
+ - reason
+ - status
+ - type
+ type: object
+ type: array
+ x-kubernetes-list-map-keys:
+ - type
+ x-kubernetes-list-type: map
+ phaseHistory:
+ description: |-
+ PhaseHistory is an append-only log of phase transitions. Unlike conditions
+ (which are updated in-place by type), this preserves the full timeline:
+ Queued → Initializing → Executing → Succeeded/Failed, each with a timestamp.
+ items:
+ description: PhaseTransition records a phase change with its timestamp.
+ properties:
+ message:
+ description: Message is an optional human-readable message about
+ the transition.
+ type: string
+ occurredAt:
+ description: OccurredAt is when this phase transition happened.
+ format: date-time
+ type: string
+ phase:
+ description: Phase is the phase that was entered (e.g. "Queued",
+ "Initializing", "Executing", "Succeeded", "Failed").
+ type: string
+ required:
+ - occurredAt
+ - phase
+ type: object
+ type: array
+ pluginPhase:
+ description: PluginPhase is a human-readable representation of the
+ plugin's current phase.
+ type: string
+ pluginPhaseVersion:
+ description: PluginPhaseVersion is the version of the current plugin
+ phase.
+ format: int32
+ type: integer
+ pluginState:
+ description: PluginState is the Gob-encoded plugin state from the
+ last reconciliation round.
+ format: byte
+ type: string
+ pluginStateVersion:
+ description: PluginStateVersion tracks the version of the plugin state
+ schema for compatibility.
+ type: integer
+ stateJson:
+ description: StateJSON is the JSON serialized NodeStatus that was
+ last sent to the State Service
+ type: string
+ type: object
+ required:
+ - spec
+ type: object
+ served: true
+ storage: true
+ subresources:
+ status: {}
diff --git a/executor/config/crd/kustomization.yaml b/executor/config/crd/kustomization.yaml
new file mode 100644
index 00000000000..eb1df2e15e3
--- /dev/null
+++ b/executor/config/crd/kustomization.yaml
@@ -0,0 +1,16 @@
+# This kustomization.yaml is not intended to be run by itself,
+# since it depends on service name and namespace that are out of this kustomize package.
+# It should be run by config/default
+resources:
+- bases/flyte.org_taskactions.yaml
+# +kubebuilder:scaffold:crdkustomizeresource
+
+patches:
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
+# patches here are for enabling the conversion webhook for each CRD
+# +kubebuilder:scaffold:crdkustomizewebhookpatch
+
+# [WEBHOOK] To enable webhook, uncomment the following section
+# the following config is for teaching kustomize how to do kustomization for CRDs.
+#configurations:
+#- kustomizeconfig.yaml
diff --git a/executor/config/crd/kustomizeconfig.yaml b/executor/config/crd/kustomizeconfig.yaml
new file mode 100644
index 00000000000..ec5c150a9df
--- /dev/null
+++ b/executor/config/crd/kustomizeconfig.yaml
@@ -0,0 +1,19 @@
+# This file is for teaching kustomize how to substitute name and namespace reference in CRD
+nameReference:
+- kind: Service
+ version: v1
+ fieldSpecs:
+ - kind: CustomResourceDefinition
+ version: v1
+ group: apiextensions.k8s.io
+ path: spec/conversion/webhook/clientConfig/service/name
+
+namespace:
+- kind: CustomResourceDefinition
+ version: v1
+ group: apiextensions.k8s.io
+ path: spec/conversion/webhook/clientConfig/service/namespace
+ create: false
+
+varReference:
+- path: metadata/annotations
diff --git a/executor/config/default/cert_metrics_manager_patch.yaml b/executor/config/default/cert_metrics_manager_patch.yaml
new file mode 100644
index 00000000000..d975015538e
--- /dev/null
+++ b/executor/config/default/cert_metrics_manager_patch.yaml
@@ -0,0 +1,30 @@
+# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs.
+
+# Add the volumeMount for the metrics-server certs
+- op: add
+ path: /spec/template/spec/containers/0/volumeMounts/-
+ value:
+ mountPath: /tmp/k8s-metrics-server/metrics-certs
+ name: metrics-certs
+ readOnly: true
+
+# Add the --metrics-cert-path argument for the metrics server
+- op: add
+ path: /spec/template/spec/containers/0/args/-
+ value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs
+
+# Add the metrics-server certs volume configuration
+- op: add
+ path: /spec/template/spec/volumes/-
+ value:
+ name: metrics-certs
+ secret:
+ secretName: metrics-server-cert
+ optional: false
+ items:
+ - key: ca.crt
+ path: ca.crt
+ - key: tls.crt
+ path: tls.crt
+ - key: tls.key
+ path: tls.key
diff --git a/executor/config/default/kustomization.yaml b/executor/config/default/kustomization.yaml
new file mode 100644
index 00000000000..07ab34a2113
--- /dev/null
+++ b/executor/config/default/kustomization.yaml
@@ -0,0 +1,234 @@
+# Adds namespace to all resources.
+namespace: executor-system
+
+# Value of this field is prepended to the
+# names of all resources, e.g. a deployment named
+# "wordpress" becomes "alices-wordpress".
+# Note that it should also match with the prefix (text before '-') of the namespace
+# field above.
+namePrefix: executor-
+
+# Labels to add to all resources and selectors.
+#labels:
+#- includeSelectors: true
+# pairs:
+# someName: someValue
+
+resources:
+- ../crd
+- ../rbac
+- ../manager
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
+# crd/kustomization.yaml
+#- ../webhook
+# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
+#- ../certmanager
+# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
+#- ../prometheus
+# [METRICS] Expose the controller manager metrics service.
+- metrics_service.yaml
+# [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy.
+# Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics.
+# Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will
+# be able to communicate with the Webhook Server.
+#- ../network-policy
+
+# Uncomment the patches line if you enable Metrics
+patches:
+# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443.
+# More info: https://book.kubebuilder.io/reference/metrics
+- path: manager_metrics_patch.yaml
+ target:
+ kind: Deployment
+
+# Uncomment the patches line if you enable Metrics and CertManager
+# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line.
+# This patch will protect the metrics with certManager self-signed certs.
+#- path: cert_metrics_manager_patch.yaml
+# target:
+# kind: Deployment
+
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
+# crd/kustomization.yaml
+#- path: manager_webhook_patch.yaml
+# target:
+# kind: Deployment
+
+# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
+# Uncomment the following replacements to add the cert-manager CA injection annotations
+#replacements:
+# - source: # Uncomment the following block to enable certificates for metrics
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.name
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+# - select: # Uncomment the following to set the Service name for TLS config in Prometheus ServiceMonitor
+# kind: ServiceMonitor
+# group: monitoring.coreos.com
+# version: v1
+# name: controller-manager-metrics-monitor
+# fieldPaths:
+# - spec.endpoints.0.tlsConfig.serverName
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+
+# - source:
+# kind: Service
+# version: v1
+# name: controller-manager-metrics-service
+# fieldPath: metadata.namespace
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: metrics-certs
+# fieldPaths:
+# - spec.dnsNames.0
+# - spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+# - select: # Uncomment the following to set the Service namespace for TLS in Prometheus ServiceMonitor
+# kind: ServiceMonitor
+# group: monitoring.coreos.com
+# version: v1
+# name: controller-manager-metrics-monitor
+# fieldPaths:
+# - spec.endpoints.0.tlsConfig.serverName
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+
+# - source: # Uncomment the following block if you have any webhook
+# kind: Service
+# version: v1
+# name: webhook-service
+# fieldPath: .metadata.name # Name of the service
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: serving-cert
+# fieldPaths:
+# - .spec.dnsNames.0
+# - .spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 0
+# create: true
+# - source:
+# kind: Service
+# version: v1
+# name: webhook-service
+# fieldPath: .metadata.namespace # Namespace of the service
+# targets:
+# - select:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: serving-cert
+# fieldPaths:
+# - .spec.dnsNames.0
+# - .spec.dnsNames.1
+# options:
+# delimiter: '.'
+# index: 1
+# create: true
+
+# - source: # Uncomment the following block if you have a ValidatingWebhook (--programmatic-validation)
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: serving-cert # This name should match the one in certificate.yaml
+# fieldPath: .metadata.namespace # Namespace of the certificate CR
+# targets:
+# - select:
+# kind: ValidatingWebhookConfiguration
+# fieldPaths:
+# - .metadata.annotations.[cert-manager.io/inject-ca-from]
+# options:
+# delimiter: '/'
+# index: 0
+# create: true
+# - source:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: serving-cert
+# fieldPath: .metadata.name
+# targets:
+# - select:
+# kind: ValidatingWebhookConfiguration
+# fieldPaths:
+# - .metadata.annotations.[cert-manager.io/inject-ca-from]
+# options:
+# delimiter: '/'
+# index: 1
+# create: true
+
+# - source: # Uncomment the following block if you have a DefaultingWebhook (--defaulting )
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: serving-cert
+# fieldPath: .metadata.namespace # Namespace of the certificate CR
+# targets:
+# - select:
+# kind: MutatingWebhookConfiguration
+# fieldPaths:
+# - .metadata.annotations.[cert-manager.io/inject-ca-from]
+# options:
+# delimiter: '/'
+# index: 0
+# create: true
+# - source:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: serving-cert
+# fieldPath: .metadata.name
+# targets:
+# - select:
+# kind: MutatingWebhookConfiguration
+# fieldPaths:
+# - .metadata.annotations.[cert-manager.io/inject-ca-from]
+# options:
+# delimiter: '/'
+# index: 1
+# create: true
+
+# - source: # Uncomment the following block if you have a ConversionWebhook (--conversion)
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: serving-cert
+# fieldPath: .metadata.namespace # Namespace of the certificate CR
+# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
+# +kubebuilder:scaffold:crdkustomizecainjectionns
+# - source:
+# kind: Certificate
+# group: cert-manager.io
+# version: v1
+# name: serving-cert
+# fieldPath: .metadata.name
+# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD.
+# +kubebuilder:scaffold:crdkustomizecainjectionname
diff --git a/executor/config/default/manager_metrics_patch.yaml b/executor/config/default/manager_metrics_patch.yaml
new file mode 100644
index 00000000000..2aaef6536f4
--- /dev/null
+++ b/executor/config/default/manager_metrics_patch.yaml
@@ -0,0 +1,4 @@
+# This patch adds the args to allow exposing the metrics endpoint using HTTPS
+- op: add
+ path: /spec/template/spec/containers/0/args/0
+ value: --metrics-bind-address=:8443
diff --git a/executor/config/default/metrics_service.yaml b/executor/config/default/metrics_service.yaml
new file mode 100644
index 00000000000..49837187deb
--- /dev/null
+++ b/executor/config/default/metrics_service.yaml
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Service
+metadata:
+ labels:
+ control-plane: controller-manager
+ app.kubernetes.io/name: executor
+ app.kubernetes.io/managed-by: kustomize
+ name: controller-manager-metrics-service
+ namespace: system
+spec:
+ ports:
+ - name: https
+ port: 8443
+ protocol: TCP
+ targetPort: 8443
+ selector:
+ control-plane: controller-manager
+ app.kubernetes.io/name: executor
diff --git a/executor/config/manager/kustomization.yaml b/executor/config/manager/kustomization.yaml
new file mode 100644
index 00000000000..ad13e96b3fc
--- /dev/null
+++ b/executor/config/manager/kustomization.yaml
@@ -0,0 +1,8 @@
+resources:
+- manager.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+images:
+- name: controller
+ newName: controller
+ newTag: latest
diff --git a/executor/config/manager/manager.yaml b/executor/config/manager/manager.yaml
new file mode 100644
index 00000000000..ff13f0329b8
--- /dev/null
+++ b/executor/config/manager/manager.yaml
@@ -0,0 +1,99 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ labels:
+ control-plane: controller-manager
+ app.kubernetes.io/name: executor
+ app.kubernetes.io/managed-by: kustomize
+ name: system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: controller-manager
+ namespace: system
+ labels:
+ control-plane: controller-manager
+ app.kubernetes.io/name: executor
+ app.kubernetes.io/managed-by: kustomize
+spec:
+ selector:
+ matchLabels:
+ control-plane: controller-manager
+ app.kubernetes.io/name: executor
+ replicas: 1
+ template:
+ metadata:
+ annotations:
+ kubectl.kubernetes.io/default-container: manager
+ labels:
+ control-plane: controller-manager
+ app.kubernetes.io/name: executor
+ spec:
+ # TODO(user): Uncomment the following code to configure the nodeAffinity expression
+ # according to the platforms which are supported by your solution.
+ # It is considered best practice to support multiple architectures. You can
+ # build your manager image using the makefile target docker-buildx.
+ # affinity:
+ # nodeAffinity:
+ # requiredDuringSchedulingIgnoredDuringExecution:
+ # nodeSelectorTerms:
+ # - matchExpressions:
+ # - key: kubernetes.io/arch
+ # operator: In
+ # values:
+ # - amd64
+ # - arm64
+ # - ppc64le
+ # - s390x
+ # - key: kubernetes.io/os
+ # operator: In
+ # values:
+ # - linux
+ securityContext:
+ # Projects are configured by default to adhere to the "restricted" Pod Security Standards.
+ # This ensures that deployments meet the highest security requirements for Kubernetes.
+ # For more details, see: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted
+ runAsNonRoot: true
+ seccompProfile:
+ type: RuntimeDefault
+ containers:
+ - command:
+ - /manager
+ args:
+ - --leader-elect
+ - --health-probe-bind-address=:8081
+ image: controller:latest
+ name: manager
+ ports: []
+ securityContext:
+ readOnlyRootFilesystem: true
+ allowPrivilegeEscalation: false
+ capabilities:
+ drop:
+ - "ALL"
+ livenessProbe:
+ httpGet:
+ path: /healthz
+ port: 8081
+ initialDelaySeconds: 15
+ periodSeconds: 20
+ readinessProbe:
+ httpGet:
+ path: /readyz
+ port: 8081
+ initialDelaySeconds: 5
+ periodSeconds: 10
+ # TODO(user): Configure the resources accordingly based on the project requirements.
+ # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+ resources:
+ limits:
+ cpu: 500m
+ memory: 128Mi
+ requests:
+ cpu: 10m
+ memory: 64Mi
+ volumeMounts: []
+ volumes: []
+ serviceAccountName: controller-manager
+ terminationGracePeriodSeconds: 10
diff --git a/executor/config/network-policy/allow-metrics-traffic.yaml b/executor/config/network-policy/allow-metrics-traffic.yaml
new file mode 100644
index 00000000000..943675cac51
--- /dev/null
+++ b/executor/config/network-policy/allow-metrics-traffic.yaml
@@ -0,0 +1,27 @@
+# This NetworkPolicy allows ingress traffic
+# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those
+# namespaces are able to gather data from the metrics endpoint.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ labels:
+ app.kubernetes.io/name: executor
+ app.kubernetes.io/managed-by: kustomize
+ name: allow-metrics-traffic
+ namespace: system
+spec:
+ podSelector:
+ matchLabels:
+ control-plane: controller-manager
+ app.kubernetes.io/name: executor
+ policyTypes:
+ - Ingress
+ ingress:
+ # This allows ingress traffic from any namespace with the label metrics: enabled
+ - from:
+ - namespaceSelector:
+ matchLabels:
+ metrics: enabled # Only from namespaces with this label
+ ports:
+ - port: 8443
+ protocol: TCP
diff --git a/executor/config/network-policy/kustomization.yaml b/executor/config/network-policy/kustomization.yaml
new file mode 100644
index 00000000000..ec0fb5e57df
--- /dev/null
+++ b/executor/config/network-policy/kustomization.yaml
@@ -0,0 +1,2 @@
+resources:
+- allow-metrics-traffic.yaml
diff --git a/executor/config/prometheus/kustomization.yaml b/executor/config/prometheus/kustomization.yaml
new file mode 100644
index 00000000000..fdc5481b103
--- /dev/null
+++ b/executor/config/prometheus/kustomization.yaml
@@ -0,0 +1,11 @@
+resources:
+- monitor.yaml
+
+# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus
+# to securely reference certificates created and managed by cert-manager.
+# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml
+# to mount the "metrics-server-cert" secret in the Manager Deployment.
+#patches:
+# - path: monitor_tls_patch.yaml
+# target:
+# kind: ServiceMonitor
diff --git a/executor/config/prometheus/monitor.yaml b/executor/config/prometheus/monitor.yaml
new file mode 100644
index 00000000000..96d85a14ce7
--- /dev/null
+++ b/executor/config/prometheus/monitor.yaml
@@ -0,0 +1,27 @@
+# Prometheus Monitor Service (Metrics)
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+ labels:
+ control-plane: controller-manager
+ app.kubernetes.io/name: executor
+ app.kubernetes.io/managed-by: kustomize
+ name: controller-manager-metrics-monitor
+ namespace: system
+spec:
+ endpoints:
+ - path: /metrics
+ port: https # Ensure this is the name of the port that exposes HTTPS metrics
+ scheme: https
+ bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+ tlsConfig:
+ # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables
+ # certificate verification, exposing the system to potential man-in-the-middle attacks.
+ # For production environments, it is recommended to use cert-manager for automatic TLS certificate management.
+ # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml,
+ # which securely references the certificate from the 'metrics-server-cert' secret.
+ insecureSkipVerify: true
+ selector:
+ matchLabels:
+ control-plane: controller-manager
+ app.kubernetes.io/name: executor
diff --git a/executor/config/prometheus/monitor_tls_patch.yaml b/executor/config/prometheus/monitor_tls_patch.yaml
new file mode 100644
index 00000000000..5bf84ce0d53
--- /dev/null
+++ b/executor/config/prometheus/monitor_tls_patch.yaml
@@ -0,0 +1,19 @@
+# Patch for Prometheus ServiceMonitor to enable secure TLS configuration
+# using certificates managed by cert-manager
+- op: replace
+ path: /spec/endpoints/0/tlsConfig
+ value:
+ # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize
+ serverName: SERVICE_NAME.SERVICE_NAMESPACE.svc
+ insecureSkipVerify: false
+ ca:
+ secret:
+ name: metrics-server-cert
+ key: ca.crt
+ cert:
+ secret:
+ name: metrics-server-cert
+ key: tls.crt
+ keySecret:
+ name: metrics-server-cert
+ key: tls.key
diff --git a/executor/config/rbac/kustomization.yaml b/executor/config/rbac/kustomization.yaml
new file mode 100644
index 00000000000..d864140ad6c
--- /dev/null
+++ b/executor/config/rbac/kustomization.yaml
@@ -0,0 +1,28 @@
+resources:
+# All RBAC will be applied under this service account in
+# the deployment namespace. You may comment out this resource
+# if your manager will use a service account that exists at
+# runtime. Be sure to update RoleBinding and ClusterRoleBinding
+# subjects if changing service account names.
+- service_account.yaml
+- role.yaml
+- role_binding.yaml
+- leader_election_role.yaml
+- leader_election_role_binding.yaml
+# The following RBAC configurations are used to protect
+# the metrics endpoint with authn/authz. These configurations
+# ensure that only authorized users and service accounts
+# can access the metrics endpoint. Comment the following
+# permissions if you want to disable this protection.
+# More info: https://book.kubebuilder.io/reference/metrics.html
+- metrics_auth_role.yaml
+- metrics_auth_role_binding.yaml
+- metrics_reader_role.yaml
+# For each CRD, "Admin", "Editor" and "Viewer" roles are scaffolded by
+# default, aiding admins in cluster management. Those roles are
+# not used by the executor itself. You can comment the following lines
+# if you do not want those helpers be installed with your Project.
+- taskaction_admin_role.yaml
+- taskaction_editor_role.yaml
+- taskaction_viewer_role.yaml
+
diff --git a/executor/config/rbac/leader_election_role.yaml b/executor/config/rbac/leader_election_role.yaml
new file mode 100644
index 00000000000..60b04c17f72
--- /dev/null
+++ b/executor/config/rbac/leader_election_role.yaml
@@ -0,0 +1,40 @@
+# permissions to do leader election.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ labels:
+ app.kubernetes.io/name: executor
+ app.kubernetes.io/managed-by: kustomize
+ name: leader-election-role
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - configmaps
+ verbs:
+ - get
+ - list
+ - watch
+ - create
+ - update
+ - patch
+ - delete
+- apiGroups:
+ - coordination.k8s.io
+ resources:
+ - leases
+ verbs:
+ - get
+ - list
+ - watch
+ - create
+ - update
+ - patch
+ - delete
+- apiGroups:
+ - ""
+ resources:
+ - events
+ verbs:
+ - create
+ - patch
diff --git a/executor/config/rbac/leader_election_role_binding.yaml b/executor/config/rbac/leader_election_role_binding.yaml
new file mode 100644
index 00000000000..38e25e51573
--- /dev/null
+++ b/executor/config/rbac/leader_election_role_binding.yaml
@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ labels:
+ app.kubernetes.io/name: executor
+ app.kubernetes.io/managed-by: kustomize
+ name: leader-election-rolebinding
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: leader-election-role
+subjects:
+- kind: ServiceAccount
+ name: controller-manager
+ namespace: system
diff --git a/executor/config/rbac/metrics_auth_role.yaml b/executor/config/rbac/metrics_auth_role.yaml
new file mode 100644
index 00000000000..32d2e4ec6b0
--- /dev/null
+++ b/executor/config/rbac/metrics_auth_role.yaml
@@ -0,0 +1,17 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: metrics-auth-role
+rules:
+- apiGroups:
+ - authentication.k8s.io
+ resources:
+ - tokenreviews
+ verbs:
+ - create
+- apiGroups:
+ - authorization.k8s.io
+ resources:
+ - subjectaccessreviews
+ verbs:
+ - create
diff --git a/executor/config/rbac/metrics_auth_role_binding.yaml b/executor/config/rbac/metrics_auth_role_binding.yaml
new file mode 100644
index 00000000000..e775d67ff08
--- /dev/null
+++ b/executor/config/rbac/metrics_auth_role_binding.yaml
@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: metrics-auth-rolebinding
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: metrics-auth-role
+subjects:
+- kind: ServiceAccount
+ name: controller-manager
+ namespace: system
diff --git a/executor/config/rbac/metrics_reader_role.yaml b/executor/config/rbac/metrics_reader_role.yaml
new file mode 100644
index 00000000000..51a75db47a5
--- /dev/null
+++ b/executor/config/rbac/metrics_reader_role.yaml
@@ -0,0 +1,9 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: metrics-reader
+rules:
+- nonResourceURLs:
+ - "/metrics"
+ verbs:
+ - get
diff --git a/executor/config/rbac/role.yaml b/executor/config/rbac/role.yaml
new file mode 100644
index 00000000000..f866ec0cda7
--- /dev/null
+++ b/executor/config/rbac/role.yaml
@@ -0,0 +1,44 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: manager-role
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - pods
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+- apiGroups:
+ - flyte.org
+ resources:
+ - taskactions
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+- apiGroups:
+ - flyte.org
+ resources:
+ - taskactions/finalizers
+ verbs:
+ - update
+- apiGroups:
+ - flyte.org
+ resources:
+ - taskactions/status
+ verbs:
+ - get
+ - patch
+ - update
diff --git a/executor/config/rbac/role_binding.yaml b/executor/config/rbac/role_binding.yaml
new file mode 100644
index 00000000000..1ccec8a9673
--- /dev/null
+++ b/executor/config/rbac/role_binding.yaml
@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ labels:
+ app.kubernetes.io/name: executor
+ app.kubernetes.io/managed-by: kustomize
+ name: manager-rolebinding
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: manager-role
+subjects:
+- kind: ServiceAccount
+ name: controller-manager
+ namespace: system
diff --git a/executor/config/rbac/service_account.yaml b/executor/config/rbac/service_account.yaml
new file mode 100644
index 00000000000..46fcf8c7f82
--- /dev/null
+++ b/executor/config/rbac/service_account.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ labels:
+ app.kubernetes.io/name: executor
+ app.kubernetes.io/managed-by: kustomize
+ name: controller-manager
+ namespace: system
diff --git a/executor/config/rbac/taskaction_admin_role.yaml b/executor/config/rbac/taskaction_admin_role.yaml
new file mode 100644
index 00000000000..efb6d4e877d
--- /dev/null
+++ b/executor/config/rbac/taskaction_admin_role.yaml
@@ -0,0 +1,27 @@
+# This rule is not used by the project executor itself.
+# It is provided to allow the cluster admin to help manage permissions for users.
+#
+# Grants full permissions ('*') over flyte.org.
+# This role is intended for users authorized to modify roles and bindings within the cluster,
+# enabling them to delegate specific permissions to other users or groups as needed.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ labels:
+ app.kubernetes.io/name: executor
+ app.kubernetes.io/managed-by: kustomize
+ name: taskaction-admin-role
+rules:
+- apiGroups:
+ - flyte.org
+ resources:
+ - taskactions
+ verbs:
+ - '*'
+- apiGroups:
+ - flyte.org
+ resources:
+ - taskactions/status
+ verbs:
+ - get
diff --git a/executor/config/rbac/taskaction_editor_role.yaml b/executor/config/rbac/taskaction_editor_role.yaml
new file mode 100644
index 00000000000..63ac23c6ea9
--- /dev/null
+++ b/executor/config/rbac/taskaction_editor_role.yaml
@@ -0,0 +1,33 @@
+# This rule is not used by the project executor itself.
+# It is provided to allow the cluster admin to help manage permissions for users.
+#
+# Grants permissions to create, update, and delete resources within the flyte.org.
+# This role is intended for users who need to manage these resources
+# but should not control RBAC or manage permissions for others.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ labels:
+ app.kubernetes.io/name: executor
+ app.kubernetes.io/managed-by: kustomize
+ name: taskaction-editor-role
+rules:
+- apiGroups:
+ - flyte.org
+ resources:
+ - taskactions
+ verbs:
+ - create
+ - delete
+ - get
+ - list
+ - patch
+ - update
+ - watch
+- apiGroups:
+ - flyte.org
+ resources:
+ - taskactions/status
+ verbs:
+ - get
diff --git a/executor/config/rbac/taskaction_viewer_role.yaml b/executor/config/rbac/taskaction_viewer_role.yaml
new file mode 100644
index 00000000000..5899d036d0c
--- /dev/null
+++ b/executor/config/rbac/taskaction_viewer_role.yaml
@@ -0,0 +1,29 @@
+# This rule is not used by the project executor itself.
+# It is provided to allow the cluster admin to help manage permissions for users.
+#
+# Grants read-only access to flyte.org resources.
+# This role is intended for users who need visibility into these resources
+# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ labels:
+ app.kubernetes.io/name: executor
+ app.kubernetes.io/managed-by: kustomize
+ name: taskaction-viewer-role
+rules:
+- apiGroups:
+ - flyte.org
+ resources:
+ - taskactions
+ verbs:
+ - get
+ - list
+ - watch
+- apiGroups:
+ - flyte.org
+ resources:
+ - taskactions/status
+ verbs:
+ - get
diff --git a/executor/config/samples/flyte.org_v1_taskaction.yaml b/executor/config/samples/flyte.org_v1_taskaction.yaml
new file mode 100644
index 00000000000..d53f870f1cf
--- /dev/null
+++ b/executor/config/samples/flyte.org_v1_taskaction.yaml
@@ -0,0 +1,15 @@
+apiVersion: flyte.org/v1
+kind: TaskAction
+metadata:
+ labels:
+ app.kubernetes.io/name: executor
+ app.kubernetes.io/managed-by: kustomize
+ name: taskaction-sample
+spec:
+ runName: "sample-run"
+ org: "demo"
+ project: "default"
+ domain: "dev"
+ actionName: "sample-task"
+ inputUri: "/tmp/input"
+ runOutputBase: "/tmp/output"
diff --git a/executor/config/samples/kustomization.yaml b/executor/config/samples/kustomization.yaml
new file mode 100644
index 00000000000..30e5acbdaa3
--- /dev/null
+++ b/executor/config/samples/kustomization.yaml
@@ -0,0 +1,4 @@
+## Append samples of your project ##
+resources:
+- flyte.org_v1_taskaction.yaml
+# +kubebuilder:scaffold:manifestskustomizesamples
diff --git a/executor/hack/boilerplate.go.txt b/executor/hack/boilerplate.go.txt
new file mode 100644
index 00000000000..221dcbe0bd9
--- /dev/null
+++ b/executor/hack/boilerplate.go.txt
@@ -0,0 +1,15 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
\ No newline at end of file
diff --git a/executor/pkg/config/config.go b/executor/pkg/config/config.go
new file mode 100644
index 00000000000..64e3ffe3952
--- /dev/null
+++ b/executor/pkg/config/config.go
@@ -0,0 +1,97 @@
+package config
+
+import (
+ "time"
+
+ stdconfig "github.com/flyteorg/flyte/v2/flytestdlib/config"
+)
+
+const configSectionKey = "executor"
+
+//go:generate pflags Config --default-var=defaultConfig
+
+var (
+ defaultConfig = &Config{
+ MetricsBindAddress: ":10254",
+ HealthProbeBindAddress: ":8081",
+ LeaderElect: false,
+ MetricsSecure: true,
+ WebhookCertName: "tls.crt",
+ WebhookCertKey: "tls.key",
+ MetricsCertName: "tls.crt",
+ MetricsCertKey: "tls.key",
+ EnableHTTP2: false,
+ EventsServiceURL: "http://localhost:8090",
+ CacheServiceURL: "http://localhost:8094",
+ Cluster: "",
+ GC: GCConfig{
+ Interval: stdconfig.Duration{Duration: 30 * time.Minute},
+ MaxTTL: stdconfig.Duration{Duration: 1 * time.Hour},
+ },
+ }
+
+ configSection = stdconfig.MustRegisterSection(configSectionKey, defaultConfig)
+)
+
+// Config holds the configuration for the executor controller manager
+type Config struct {
+ // MetricsBindAddress is the address the metrics endpoint binds to.
+ // Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable.
+ MetricsBindAddress string `json:"metricsBindAddress" pflag:",Address the metrics endpoint binds to"`
+
+ // HealthProbeBindAddress is the address the probe endpoint binds to.
+ HealthProbeBindAddress string `json:"healthProbeBindAddress" pflag:",Address the probe endpoint binds to"`
+
+ // LeaderElect enables leader election for controller manager.
+ LeaderElect bool `json:"leaderElect" pflag:",Enable leader election for controller manager"`
+
+ // MetricsSecure controls whether the metrics endpoint is served via HTTPS.
+ MetricsSecure bool `json:"metricsSecure" pflag:",Serve metrics endpoint via HTTPS"`
+
+ // WebhookCertPath is the directory containing the webhook certificate.
+ WebhookCertPath string `json:"webhookCertPath" pflag:",Directory containing the webhook certificate"`
+
+ // WebhookCertName is the name of the webhook certificate file.
+ WebhookCertName string `json:"webhookCertName" pflag:",Name of the webhook certificate file"`
+
+ // WebhookCertKey is the name of the webhook key file.
+ WebhookCertKey string `json:"webhookCertKey" pflag:",Name of the webhook key file"`
+
+ // MetricsCertPath is the directory containing the metrics server certificate.
+ MetricsCertPath string `json:"metricsCertPath" pflag:",Directory containing the metrics server certificate"`
+
+ // MetricsCertName is the name of the metrics server certificate file.
+ MetricsCertName string `json:"metricsCertName" pflag:",Name of the metrics server certificate file"`
+
+ // MetricsCertKey is the name of the metrics server key file.
+ MetricsCertKey string `json:"metricsCertKey" pflag:",Name of the metrics server key file"`
+
+ // EnableHTTP2 enables HTTP/2 for the metrics and webhook servers.
+ EnableHTTP2 bool `json:"enableHTTP2" pflag:",Enable HTTP/2 for metrics and webhook servers"`
+
+ // EventsServiceURL is the URL of the event Service for reporting action state updates.
+ EventsServiceURL string `json:"EventsServiceURL" pflag:",URL of the Event Service for action event updates"`
+
+ // CacheServiceURL is the URL of the cache service for catalog operations.
+ CacheServiceURL string `json:"cacheServiceURL" pflag:",URL of the cache service for task cache operations"`
+
+ // Cluster is the cluster identifier attached to action events.
+ Cluster string `json:"cluster" pflag:",Cluster identifier for action events"`
+
+ // GC configures the garbage collector for terminal TaskActions.
+ GC GCConfig `json:"gc" pflag:",Garbage collector configuration for terminal TaskActions"`
+}
+
+// GCConfig holds the configuration for the TaskAction garbage collector.
+type GCConfig struct {
+ // Interval is how often the garbage collector runs. 0 disables GC.
+ Interval stdconfig.Duration `json:"interval" pflag:",How often the garbage collector runs. 0 disables GC."`
+
+ // MaxTTL is the time-to-live for terminal TaskActions before deletion.
+ MaxTTL stdconfig.Duration `json:"maxTTL" pflag:",Time-to-live for terminal TaskActions before deletion."`
+}
+
+// GetConfig returns the parsed executor configuration
+func GetConfig() *Config {
+ return configSection.GetConfig().(*Config)
+}
diff --git a/executor/pkg/config/config_flags.go b/executor/pkg/config/config_flags.go
new file mode 100755
index 00000000000..f7378733153
--- /dev/null
+++ b/executor/pkg/config/config_flags.go
@@ -0,0 +1,69 @@
+// Code generated by go generate; DO NOT EDIT.
+// This file was generated by robots.
+
+package config
+
+import (
+ "encoding/json"
+ "reflect"
+
+ "fmt"
+
+ "github.com/spf13/pflag"
+)
+
+// If v is a pointer, it will get its element value or the zero value of the element type.
+// If v is not a pointer, it will return it as is.
+func (Config) elemValueOrNil(v interface{}) interface{} {
+ if t := reflect.TypeOf(v); t.Kind() == reflect.Ptr {
+ if reflect.ValueOf(v).IsNil() {
+ return reflect.Zero(t.Elem()).Interface()
+ } else {
+ return reflect.ValueOf(v).Interface()
+ }
+ } else if v == nil {
+ return reflect.Zero(t).Interface()
+ }
+
+ return v
+}
+
+func (Config) mustJsonMarshal(v interface{}) string {
+ raw, err := json.Marshal(v)
+ if err != nil {
+ panic(err)
+ }
+
+ return string(raw)
+}
+
+func (Config) mustMarshalJSON(v json.Marshaler) string {
+ raw, err := v.MarshalJSON()
+ if err != nil {
+ panic(err)
+ }
+
+ return string(raw)
+}
+
+// GetPFlagSet will return strongly types pflags for all fields in Config and its nested types. The format of the
+// flags is json-name.json-sub-name... etc.
+func (cfg Config) GetPFlagSet(prefix string) *pflag.FlagSet {
+ cmdFlags := pflag.NewFlagSet("Config", pflag.ExitOnError)
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "metricsBindAddress"), defaultConfig.MetricsBindAddress, "Address the metrics endpoint binds to")
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "healthProbeBindAddress"), defaultConfig.HealthProbeBindAddress, "Address the probe endpoint binds to")
+ cmdFlags.Bool(fmt.Sprintf("%v%v", prefix, "leaderElect"), defaultConfig.LeaderElect, "Enable leader election for controller manager")
+ cmdFlags.Bool(fmt.Sprintf("%v%v", prefix, "metricsSecure"), defaultConfig.MetricsSecure, "Serve metrics endpoint via HTTPS")
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "webhookCertPath"), defaultConfig.WebhookCertPath, "Directory containing the webhook certificate")
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "webhookCertName"), defaultConfig.WebhookCertName, "Name of the webhook certificate file")
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "webhookCertKey"), defaultConfig.WebhookCertKey, "Name of the webhook key file")
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "metricsCertPath"), defaultConfig.MetricsCertPath, "Directory containing the metrics server certificate")
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "metricsCertName"), defaultConfig.MetricsCertName, "Name of the metrics server certificate file")
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "metricsCertKey"), defaultConfig.MetricsCertKey, "Name of the metrics server key file")
+ cmdFlags.Bool(fmt.Sprintf("%v%v", prefix, "enableHTTP2"), defaultConfig.EnableHTTP2, "Enable HTTP/2 for metrics and webhook servers")
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "EventsServiceURL"), defaultConfig.EventsServiceURL, "URL of the Event Service for action event updates")
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "cluster"), defaultConfig.Cluster, "Cluster identifier for action events")
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "gc.interval"), defaultConfig.GC.Interval.String(), "How often the garbage collector runs. 0 disables GC.")
+ cmdFlags.String(fmt.Sprintf("%v%v", prefix, "gc.maxTTL"), defaultConfig.GC.MaxTTL.String(), "Time-to-live for terminal TaskActions before deletion.")
+ return cmdFlags
+}
diff --git a/executor/pkg/config/config_flags_test.go b/executor/pkg/config/config_flags_test.go
new file mode 100755
index 00000000000..363ec7c3c5c
--- /dev/null
+++ b/executor/pkg/config/config_flags_test.go
@@ -0,0 +1,312 @@
+// Code generated by go generate; DO NOT EDIT.
+// This file was generated by robots.
+
+package config
+
+import (
+ "encoding/json"
+ "fmt"
+ "reflect"
+ "strings"
+ "testing"
+
+ "github.com/mitchellh/mapstructure"
+ "github.com/stretchr/testify/assert"
+)
+
+var dereferencableKindsConfig = map[reflect.Kind]struct{}{
+ reflect.Array: {}, reflect.Chan: {}, reflect.Map: {}, reflect.Ptr: {}, reflect.Slice: {},
+}
+
+// Checks if t is a kind that can be dereferenced to get its underlying type.
+func canGetElementConfig(t reflect.Kind) bool {
+ _, exists := dereferencableKindsConfig[t]
+ return exists
+}
+
+// This decoder hook tests types for json unmarshaling capability. If implemented, it uses json unmarshal to build the
+// object. Otherwise, it'll just pass on the original data.
+func jsonUnmarshalerHookConfig(_, to reflect.Type, data interface{}) (interface{}, error) {
+ unmarshalerType := reflect.TypeOf((*json.Unmarshaler)(nil)).Elem()
+ if to.Implements(unmarshalerType) || reflect.PtrTo(to).Implements(unmarshalerType) ||
+ (canGetElementConfig(to.Kind()) && to.Elem().Implements(unmarshalerType)) {
+
+ raw, err := json.Marshal(data)
+ if err != nil {
+ fmt.Printf("Failed to marshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err)
+ return data, nil
+ }
+
+ res := reflect.New(to).Interface()
+ err = json.Unmarshal(raw, &res)
+ if err != nil {
+ fmt.Printf("Failed to umarshal Data: %v. Error: %v. Skipping jsonUnmarshalHook", data, err)
+ return data, nil
+ }
+
+ return res, nil
+ }
+
+ return data, nil
+}
+
+func decode_Config(input, result interface{}) error {
+ config := &mapstructure.DecoderConfig{
+ TagName: "json",
+ WeaklyTypedInput: true,
+ Result: result,
+ DecodeHook: mapstructure.ComposeDecodeHookFunc(
+ mapstructure.StringToTimeDurationHookFunc(),
+ mapstructure.StringToSliceHookFunc(","),
+ jsonUnmarshalerHookConfig,
+ ),
+ }
+
+ decoder, err := mapstructure.NewDecoder(config)
+ if err != nil {
+ return err
+ }
+
+ return decoder.Decode(input)
+}
+
+func join_Config(arr interface{}, sep string) string {
+ listValue := reflect.ValueOf(arr)
+ strs := make([]string, 0, listValue.Len())
+ for i := 0; i < listValue.Len(); i++ {
+ strs = append(strs, fmt.Sprintf("%v", listValue.Index(i)))
+ }
+
+ return strings.Join(strs, sep)
+}
+
+func testDecodeJson_Config(t *testing.T, val, result interface{}) {
+ assert.NoError(t, decode_Config(val, result))
+}
+
+func testDecodeRaw_Config(t *testing.T, vStringSlice, result interface{}) {
+ assert.NoError(t, decode_Config(vStringSlice, result))
+}
+
+func TestConfig_GetPFlagSet(t *testing.T) {
+ val := Config{}
+ cmdFlags := val.GetPFlagSet("")
+ assert.True(t, cmdFlags.HasFlags())
+}
+
+func TestConfig_SetFlags(t *testing.T) {
+ actual := Config{}
+ cmdFlags := actual.GetPFlagSet("")
+ assert.True(t, cmdFlags.HasFlags())
+
+ t.Run("Test_metricsBindAddress", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("metricsBindAddress", testValue)
+ if vString, err := cmdFlags.GetString("metricsBindAddress"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.MetricsBindAddress)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_healthProbeBindAddress", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("healthProbeBindAddress", testValue)
+ if vString, err := cmdFlags.GetString("healthProbeBindAddress"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.HealthProbeBindAddress)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_leaderElect", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("leaderElect", testValue)
+ if vBool, err := cmdFlags.GetBool("leaderElect"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vBool), &actual.LeaderElect)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_metricsSecure", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("metricsSecure", testValue)
+ if vBool, err := cmdFlags.GetBool("metricsSecure"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vBool), &actual.MetricsSecure)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_webhookCertPath", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("webhookCertPath", testValue)
+ if vString, err := cmdFlags.GetString("webhookCertPath"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.WebhookCertPath)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_webhookCertName", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("webhookCertName", testValue)
+ if vString, err := cmdFlags.GetString("webhookCertName"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.WebhookCertName)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_webhookCertKey", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("webhookCertKey", testValue)
+ if vString, err := cmdFlags.GetString("webhookCertKey"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.WebhookCertKey)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_metricsCertPath", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("metricsCertPath", testValue)
+ if vString, err := cmdFlags.GetString("metricsCertPath"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.MetricsCertPath)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_metricsCertName", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("metricsCertName", testValue)
+ if vString, err := cmdFlags.GetString("metricsCertName"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.MetricsCertName)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_metricsCertKey", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("metricsCertKey", testValue)
+ if vString, err := cmdFlags.GetString("metricsCertKey"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.MetricsCertKey)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_enableHTTP2", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("enableHTTP2", testValue)
+ if vBool, err := cmdFlags.GetBool("enableHTTP2"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vBool), &actual.EnableHTTP2)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_EventsServiceURL", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("EventsServiceURL", testValue)
+ if vString, err := cmdFlags.GetString("EventsServiceURL"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.EventsServiceURL)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_cluster", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := "1"
+
+ cmdFlags.Set("cluster", testValue)
+ if vString, err := cmdFlags.GetString("cluster"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.Cluster)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_gc.interval", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := defaultConfig.GC.Interval.String()
+
+ cmdFlags.Set("gc.interval", testValue)
+ if vString, err := cmdFlags.GetString("gc.interval"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.GC.Interval)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+ t.Run("Test_gc.maxTTL", func(t *testing.T) {
+
+ t.Run("Override", func(t *testing.T) {
+ testValue := defaultConfig.GC.MaxTTL.String()
+
+ cmdFlags.Set("gc.maxTTL", testValue)
+ if vString, err := cmdFlags.GetString("gc.maxTTL"); err == nil {
+ testDecodeJson_Config(t, fmt.Sprintf("%v", vString), &actual.GC.MaxTTL)
+
+ } else {
+ assert.FailNow(t, err.Error())
+ }
+ })
+ })
+}
diff --git a/executor/pkg/controller/garbage_collector.go b/executor/pkg/controller/garbage_collector.go
new file mode 100644
index 00000000000..8429fb471b9
--- /dev/null
+++ b/executor/pkg/controller/garbage_collector.go
@@ -0,0 +1,109 @@
+package controller
+
+import (
+ "context"
+ "time"
+
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/log"
+
+ flyteorgv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+)
+
+// GarbageCollector periodically deletes terminal TaskActions that have exceeded their TTL.
+// It implements the controller-runtime manager.Runnable interface.
+type GarbageCollector struct {
+ client client.Client
+ interval time.Duration
+ maxTTL time.Duration
+}
+
+// NewGarbageCollector creates a new GarbageCollector.
+func NewGarbageCollector(c client.Client, interval, maxTTL time.Duration) *GarbageCollector {
+ return &GarbageCollector{
+ client: c,
+ interval: interval,
+ maxTTL: maxTTL,
+ }
+}
+
+// Start runs the garbage collection loop until the context is cancelled.
+// It satisfies the manager.Runnable interface.
+func (gc *GarbageCollector) Start(ctx context.Context) error {
+ logger := log.FromContext(ctx).WithName("gc")
+ logger.Info("starting TaskAction garbage collector", "interval", gc.interval, "maxTTL", gc.maxTTL)
+
+ ticker := time.NewTicker(gc.interval)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ logger.Info("stopping TaskAction garbage collector")
+ return nil
+ case <-ticker.C:
+ if err := gc.collect(ctx); err != nil {
+ logger.Error(err, "garbage collection cycle failed")
+ }
+ }
+ }
+}
+
+const gcPageSize = 500
+
+// collect lists all terminated TaskActions (paginated) and deletes those whose completed-time has expired.
+func (gc *GarbageCollector) collect(ctx context.Context) error {
+ logger := log.FromContext(ctx).WithName("gc")
+
+ cutoff := time.Now().UTC().Add(-gc.maxTTL).Format(labelTimeFormat)
+ deleted := 0
+ total := 0
+ continueToken := ""
+
+ for {
+ var taskActions flyteorgv1.TaskActionList
+ listOpts := []client.ListOption{
+ client.MatchingLabels{LabelTerminationStatus: LabelValueTerminated},
+ client.HasLabels{LabelCompletedTime},
+ client.Limit(gcPageSize),
+ }
+ if continueToken != "" {
+ listOpts = append(listOpts, client.Continue(continueToken))
+ }
+
+ if err := gc.client.List(ctx, &taskActions, listOpts...); err != nil {
+ return err
+ }
+
+ total += len(taskActions.Items)
+
+ for i := range taskActions.Items {
+ ta := &taskActions.Items[i]
+ completedTime := ta.GetLabels()[LabelCompletedTime]
+ if completedTime == "" {
+ continue
+ }
+
+ // The minute-precision format is lexicographically ordered, so string comparison works.
+ if completedTime < cutoff {
+ if err := gc.client.Delete(ctx, ta); err != nil {
+ logger.Error(err, "failed to delete expired TaskAction",
+ "name", ta.Name, "namespace", ta.Namespace, "completedTime", completedTime)
+ continue
+ }
+ deleted++
+ }
+ }
+
+ continueToken = taskActions.GetContinue()
+ if continueToken == "" {
+ break
+ }
+ }
+
+ if deleted > 0 {
+ logger.Info("garbage collection completed", "deleted", deleted, "total", total)
+ }
+
+ return nil
+}
diff --git a/executor/pkg/controller/garbage_collector_test.go b/executor/pkg/controller/garbage_collector_test.go
new file mode 100644
index 00000000000..c2a967726e1
--- /dev/null
+++ b/executor/pkg/controller/garbage_collector_test.go
@@ -0,0 +1,151 @@
+package controller
+
+import (
+ "context"
+ "time"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+
+ flyteorgv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+)
+
+func createTaskAction(ctx context.Context, name string, labels map[string]string) *flyteorgv1.TaskAction {
+ ta := &flyteorgv1.TaskAction{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: "default",
+ Labels: labels,
+ },
+ Spec: flyteorgv1.TaskActionSpec{
+ RunName: "test-run",
+ Project: "test-project",
+ Domain: "test-domain",
+ ActionName: "test-action",
+ InputURI: "/tmp/input",
+ RunOutputBase: "/tmp/output",
+ TaskType: "python-task",
+ TaskTemplate: buildTaskTemplateBytes("python-task", "python:3.11"),
+ },
+ }
+ ExpectWithOffset(1, k8sClient.Create(ctx, ta)).To(Succeed())
+ return ta
+}
+
+func deleteTaskAction(ctx context.Context, name string) {
+ ta := &flyteorgv1.TaskAction{}
+ err := k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: "default"}, ta)
+ if err == nil {
+ _ = k8sClient.Delete(ctx, ta)
+ }
+}
+
+var _ = Describe("GarbageCollector", func() {
+ ctx := context.Background()
+
+ AfterEach(func() {
+ // Clean up all TaskActions in default namespace
+ var list flyteorgv1.TaskActionList
+ Expect(k8sClient.List(ctx, &list, client.InNamespace("default"))).To(Succeed())
+ for i := range list.Items {
+ _ = k8sClient.Delete(ctx, &list.Items[i])
+ }
+ })
+
+ It("should delete TaskActions with expired completed-time label", func() {
+ expiredTime := time.Now().UTC().Add(-2 * time.Hour).Format(labelTimeFormat)
+ createTaskAction(ctx, "gc-expired", map[string]string{
+ LabelTerminationStatus: LabelValueTerminated,
+ LabelCompletedTime: expiredTime,
+ })
+
+ gc := NewGarbageCollector(k8sClient, 1*time.Minute, 1*time.Hour)
+ Expect(gc.collect(ctx)).To(Succeed())
+
+ ta := &flyteorgv1.TaskAction{}
+ err := k8sClient.Get(ctx, types.NamespacedName{Name: "gc-expired", Namespace: "default"}, ta)
+ Expect(err).To(HaveOccurred())
+ Expect(client.IgnoreNotFound(err)).To(Succeed())
+ })
+
+ It("should retain TaskActions with recent completed-time label", func() {
+ recentTime := time.Now().UTC().Format(labelTimeFormat)
+ createTaskAction(ctx, "gc-recent", map[string]string{
+ LabelTerminationStatus: LabelValueTerminated,
+ LabelCompletedTime: recentTime,
+ })
+
+ gc := NewGarbageCollector(k8sClient, 1*time.Minute, 1*time.Hour)
+ Expect(gc.collect(ctx)).To(Succeed())
+
+ ta := &flyteorgv1.TaskAction{}
+ err := k8sClient.Get(ctx, types.NamespacedName{Name: "gc-recent", Namespace: "default"}, ta)
+ Expect(err).NotTo(HaveOccurred())
+ })
+
+ It("should retain non-terminated TaskActions", func() {
+ createTaskAction(ctx, "gc-active", nil)
+
+ gc := NewGarbageCollector(k8sClient, 1*time.Minute, 1*time.Hour)
+ Expect(gc.collect(ctx)).To(Succeed())
+
+ ta := &flyteorgv1.TaskAction{}
+ err := k8sClient.Get(ctx, types.NamespacedName{Name: "gc-active", Namespace: "default"}, ta)
+ Expect(err).NotTo(HaveOccurred())
+ })
+
+ It("should handle empty list gracefully", func() {
+ gc := NewGarbageCollector(k8sClient, 1*time.Minute, 1*time.Hour)
+ Expect(gc.collect(ctx)).To(Succeed())
+ })
+})
+
+var _ = Describe("ensureTerminalLabels", func() {
+ ctx := context.Background()
+
+ AfterEach(func() {
+ deleteTaskAction(ctx, "terminal-labels-test")
+ })
+
+ It("should patch completed-time when termination-status is set but completed-time is missing", func() {
+ ta := createTaskAction(ctx, "terminal-missing-time", map[string]string{
+ LabelTerminationStatus: LabelValueTerminated,
+ })
+ defer deleteTaskAction(ctx, "terminal-missing-time")
+
+ reconciler := &TaskActionReconciler{
+ Client: k8sClient,
+ Scheme: k8sClient.Scheme(),
+ }
+
+ Expect(reconciler.ensureTerminalLabels(ctx, ta)).To(Succeed())
+ Expect(ta.GetLabels()[LabelTerminationStatus]).To(Equal(LabelValueTerminated))
+ Expect(ta.GetLabels()[LabelCompletedTime]).NotTo(BeEmpty())
+ })
+
+ It("should be idempotent", func() {
+ ta := createTaskAction(ctx, "terminal-labels-test", nil)
+
+ reconciler := &TaskActionReconciler{
+ Client: k8sClient,
+ Scheme: k8sClient.Scheme(),
+ }
+
+ // First call should set labels
+ Expect(reconciler.ensureTerminalLabels(ctx, ta)).To(Succeed())
+ Expect(ta.GetLabels()[LabelTerminationStatus]).To(Equal(LabelValueTerminated))
+ Expect(ta.GetLabels()[LabelCompletedTime]).NotTo(BeEmpty())
+ firstCompletedTime := ta.GetLabels()[LabelCompletedTime]
+
+ // Re-fetch to get updated resource version
+ updated := &flyteorgv1.TaskAction{}
+ Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "terminal-labels-test", Namespace: "default"}, updated)).To(Succeed())
+
+ // Second call should be a no-op (labels already set)
+ Expect(reconciler.ensureTerminalLabels(ctx, updated)).To(Succeed())
+ Expect(updated.GetLabels()[LabelCompletedTime]).To(Equal(firstCompletedTime))
+ })
+})
diff --git a/executor/pkg/controller/output_refs_test.go b/executor/pkg/controller/output_refs_test.go
new file mode 100644
index 00000000000..68ce4897eff
--- /dev/null
+++ b/executor/pkg/controller/output_refs_test.go
@@ -0,0 +1,97 @@
+package controller
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+ flyteorgv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+)
+
+func taskActionForOutputRefs(namespace, name, runOutputBase, actionName string, attempts uint32) *flyteorgv1.TaskAction {
+ return &flyteorgv1.TaskAction{
+ ObjectMeta: metav1.ObjectMeta{
+ Namespace: namespace,
+ Name: name,
+ },
+ Spec: flyteorgv1.TaskActionSpec{
+ RunOutputBase: runOutputBase,
+ ActionName: actionName,
+ },
+ Status: flyteorgv1.TaskActionStatus{
+ Attempts: attempts,
+ },
+ }
+}
+
+func TestOutputRefs(t *testing.T) {
+ ctx := context.Background()
+
+ t.Run("empty RunOutputBase returns nil", func(t *testing.T) {
+ ta := taskActionForOutputRefs("flyte", "ta-abc", "", "action-0", 1)
+ assert.Nil(t, outputRefs(ctx, ta))
+ })
+
+ t.Run("output URI ends with outputs.pb", func(t *testing.T) {
+ ta := taskActionForOutputRefs("flyte", "ta-abc", "s3://my-bucket/org/proj/dev/run123", "action-0", 1)
+ refs := outputRefs(ctx, ta)
+ require.NotNil(t, refs)
+ assert.True(t, strings.HasSuffix(refs.GetOutputUri(), "/outputs.pb"),
+ "expected URI to end with /outputs.pb, got: %s", refs.GetOutputUri())
+ })
+
+ t.Run("output URI contains action name", func(t *testing.T) {
+ ta := taskActionForOutputRefs("flyte", "ta-abc", "s3://my-bucket/org/proj/dev/run123", "action-0", 1)
+ refs := outputRefs(ctx, ta)
+ require.NotNil(t, refs)
+ assert.Contains(t, refs.GetOutputUri(), "action-0")
+ })
+
+ t.Run("output URI contains attempt number", func(t *testing.T) {
+ ta := taskActionForOutputRefs("flyte", "ta-abc", "s3://my-bucket/org/proj/dev/run123", "action-0", 2)
+ refs := outputRefs(ctx, ta)
+ require.NotNil(t, refs)
+ // segment before outputs.pb should be the attempt number
+ uri := strings.TrimSuffix(refs.GetOutputUri(), "/outputs.pb")
+ parts := strings.Split(uri, "/")
+ assert.Equal(t, "2", parts[len(parts)-1])
+ })
+
+ t.Run("attempt defaults to 1 when Status.Attempts is zero", func(t *testing.T) {
+ ta := taskActionForOutputRefs("flyte", "ta-abc", "s3://my-bucket/org/proj/dev/run123", "action-0", 0)
+ refs := outputRefs(ctx, ta)
+ require.NotNil(t, refs)
+ uri := strings.TrimSuffix(refs.GetOutputUri(), "/outputs.pb")
+ parts := strings.Split(uri, "/")
+ assert.Equal(t, "1", parts[len(parts)-1])
+ })
+
+ t.Run("report URI ends with report.html and shares prefix with output URI", func(t *testing.T) {
+ ta := taskActionForOutputRefs("flyte", "ta-abc", "s3://my-bucket/org/proj/dev/run123", "action-0", 1)
+ refs := outputRefs(ctx, ta)
+ require.NotNil(t, refs)
+ assert.True(t, strings.HasSuffix(refs.GetReportUri(), "/report.html"),
+ "expected report URI to end with /report.html, got: %s", refs.GetReportUri())
+ assert.Equal(t,
+ strings.TrimSuffix(refs.GetOutputUri(), "/outputs.pb"),
+ strings.TrimSuffix(refs.GetReportUri(), "/report.html"),
+ "report URI and output URI should share the same prefix")
+ })
+
+ t.Run("output URI is consistent with ComputeActionOutputPath", func(t *testing.T) {
+ // outputRefs and NewTaskExecutionContext must agree on the output path.
+ // Verify outputRefs produces a URI whose directory matches the plugin's output prefix.
+ ta := taskActionForOutputRefs("flyte", "ta-abc", "s3://my-bucket/org/proj/dev/run123", "action-0", 1)
+ refs := outputRefs(ctx, ta)
+ require.NotNil(t, refs)
+
+ // The URI should be /outputs.pb — strip the file to get the prefix.
+ dir := strings.TrimSuffix(refs.GetOutputUri(), "/outputs.pb")
+ assert.NotEmpty(t, dir)
+ assert.NotEqual(t, refs.GetOutputUri(), dir, "TrimSuffix should have removed /outputs.pb")
+ })
+}
\ No newline at end of file
diff --git a/executor/pkg/controller/suite_test.go b/executor/pkg/controller/suite_test.go
new file mode 100644
index 00000000000..94681a28628
--- /dev/null
+++ b/executor/pkg/controller/suite_test.go
@@ -0,0 +1,160 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ "k8s.io/client-go/kubernetes/scheme"
+ "k8s.io/client-go/rest"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/envtest"
+ logf "sigs.k8s.io/controller-runtime/pkg/log"
+ "sigs.k8s.io/controller-runtime/pkg/log/zap"
+ metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+
+ flyteorgv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+ "github.com/flyteorg/flyte/v2/executor/pkg/plugin"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+
+ // Register the pod plugin so the registry can resolve container/python task types.
+ _ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/plugins/k8s/pod"
+)
+
+// These tests use Ginkgo (BDD-style Go testing framework). Refer to
+// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
+
+var (
+ ctx context.Context
+ cancel context.CancelFunc
+ testEnv *envtest.Environment
+ cfg *rest.Config
+ k8sClient client.Client
+ pluginRegistry *plugin.Registry
+ dataStore *storage.DataStore
+)
+
+func TestControllers(t *testing.T) {
+ RegisterFailHandler(Fail)
+
+ RunSpecs(t, "Controller Suite")
+}
+
+var _ = BeforeSuite(func() {
+ logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
+
+ ctx, cancel = context.WithCancel(context.TODO())
+
+ var err error
+ err = flyteorgv1.AddToScheme(scheme.Scheme)
+ Expect(err).NotTo(HaveOccurred())
+
+ // +kubebuilder:scaffold:scheme
+
+ By("bootstrapping test environment")
+ testEnv = &envtest.Environment{
+ CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")},
+ ErrorIfCRDPathMissing: true,
+ }
+
+ // Retrieve the first found binary directory to allow running tests from IDEs
+ if getFirstFoundEnvTestBinaryDir() != "" {
+ testEnv.BinaryAssetsDirectory = getFirstFoundEnvTestBinaryDir()
+ }
+
+ // cfg is defined in this file globally.
+ cfg, err = testEnv.Start()
+ Expect(err).NotTo(HaveOccurred())
+ Expect(cfg).NotTo(BeNil())
+
+ k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
+ Expect(err).NotTo(HaveOccurred())
+ Expect(k8sClient).NotTo(BeNil())
+
+ By("setting up controller-runtime manager for plugin registry")
+ mgr, err := ctrl.NewManager(cfg, ctrl.Options{
+ Scheme: scheme.Scheme,
+ Metrics: metricsserver.Options{
+ BindAddress: "0", // disable metrics server in tests
+ },
+ })
+ Expect(err).NotTo(HaveOccurred())
+
+ // Start manager in background — needed so the cache syncs and
+ // kubeClient.GetClient()/GetCache() work for plugins.
+ go func() {
+ defer GinkgoRecover()
+ Expect(mgr.Start(ctx)).To(Succeed())
+ }()
+
+ // Wait for the cache to sync before proceeding.
+ Expect(mgr.GetCache().WaitForCacheSync(ctx)).To(BeTrue())
+
+ By("initializing plugin registry with pod plugin")
+ setupCtx := plugin.NewSetupContext(
+ mgr, nil, nil, nil, nil,
+ "TaskAction",
+ promutils.NewScope("test"),
+ )
+ pluginRegistry = plugin.NewRegistry(setupCtx, pluginmachinery.PluginRegistry())
+ Expect(pluginRegistry.Initialize(ctx)).To(Succeed())
+
+ By("initializing labeled metrics keys")
+ ensureTestMetricKeys()
+
+ By("creating in-memory data store")
+ dataStore, err = storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, promutils.NewScope("test:storage"))
+ Expect(err).NotTo(HaveOccurred())
+})
+
+var _ = AfterSuite(func() {
+ By("tearing down the test environment")
+ cancel()
+ err := testEnv.Stop()
+ Expect(err).NotTo(HaveOccurred())
+})
+
+// getFirstFoundEnvTestBinaryDir locates the first binary in the specified path.
+// ENVTEST-based tests depend on specific binaries, usually located in paths set by
+// controller-runtime. When running tests directly (e.g., via an IDE) without using
+// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured.
+//
+// This function streamlines the process by finding the required binaries, similar to
+// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are
+// properly set up, run 'make setup-envtest' beforehand.
+func getFirstFoundEnvTestBinaryDir() string {
+ basePath := filepath.Join("..", "..", "bin", "k8s")
+ entries, err := os.ReadDir(basePath)
+ if err != nil {
+ logf.Log.Error(err, "Failed to read directory", "path", basePath)
+ return ""
+ }
+ for _, entry := range entries {
+ if entry.IsDir() {
+ return filepath.Join(basePath, entry.Name())
+ }
+ }
+ return ""
+}
diff --git a/executor/pkg/controller/taskaction_cache.go b/executor/pkg/controller/taskaction_cache.go
new file mode 100644
index 00000000000..e68d0483bf0
--- /dev/null
+++ b/executor/pkg/controller/taskaction_cache.go
@@ -0,0 +1,206 @@
+package controller
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "google.golang.org/grpc/codes"
+ grpcstatus "google.golang.org/grpc/status"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/timestamppb"
+ "k8s.io/apimachinery/pkg/types"
+ "sigs.k8s.io/controller-runtime/pkg/log"
+
+ flyteorgv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/catalog"
+ pluginsCore "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/core"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/ioutils"
+ corepb "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+const cacheReservationHeartbeatInterval = TaskActionDefaultRequeueDuration
+
+type taskCacheConfig struct {
+ key catalog.Key
+ ownerID string
+ serializable bool
+}
+
+func (r *TaskActionReconciler) evaluateCacheBeforeExecution(
+ ctx context.Context,
+ taskAction *flyteorgv1.TaskAction,
+ tCtx pluginsCore.TaskExecutionContext,
+) (pluginsCore.Transition, bool, error) {
+ cacheCfg, ok, err := buildTaskCacheConfig(ctx, taskAction, tCtx)
+ if err != nil || !ok || r.Catalog == nil {
+ return pluginsCore.UnknownTransition, false, err
+ }
+
+ entry, err := r.Catalog.Get(ctx, cacheCfg.key)
+ if err == nil {
+ if err := tCtx.OutputWriter().Put(ctx, entry.GetOutputs()); err != nil {
+ return pluginsCore.UnknownTransition, false, fmt.Errorf("persisting cached outputs: %w", err)
+ }
+
+ info := cacheTaskInfo(corepb.CatalogCacheStatus_CACHE_HIT, "cache hit")
+ return pluginsCore.DoTransition(pluginsCore.PhaseInfoSuccess(info)), true, nil
+ }
+
+ if !catalog.IsNotFound(err) {
+ log.FromContext(ctx).Error(err, "cache lookup failed, continuing with task execution")
+ return pluginsCore.UnknownTransition, false, nil
+ }
+
+ if cacheCfg.serializable {
+ reservation, err := r.Catalog.GetOrExtendReservation(ctx, cacheCfg.key, cacheCfg.ownerID, cacheReservationHeartbeatInterval)
+ if err != nil {
+ return pluginsCore.UnknownTransition, false, fmt.Errorf("acquiring cache reservation: %w", err)
+ }
+ if reservation.GetOwnerId() == cacheCfg.ownerID {
+ return pluginsCore.UnknownTransition, false, nil
+ }
+
+ info := cacheTaskInfo(corepb.CatalogCacheStatus_CACHE_MISS, "waiting for serialized cache owner")
+ phaseInfo := pluginsCore.PhaseInfoWaitingForCache(taskAction.Status.PluginPhaseVersion, info)
+ phaseInfo.WithReason(fmt.Sprintf("waiting for cache to be populated by reservation owner %q", reservation.GetOwnerId()))
+ return pluginsCore.DoTransition(phaseInfo), true, nil
+ }
+
+ return pluginsCore.UnknownTransition, false, nil
+}
+
+func (r *TaskActionReconciler) finalizeCacheAfterExecution(
+ ctx context.Context,
+ taskAction *flyteorgv1.TaskAction,
+ tCtx pluginsCore.TaskExecutionContext,
+ transition pluginsCore.Transition,
+ cacheShortCircuited bool,
+) (pluginsCore.Transition, error) {
+ cacheCfg, ok, err := buildTaskCacheConfig(ctx, taskAction, tCtx)
+ if err != nil || !ok || r.Catalog == nil {
+ return transition, err
+ }
+
+ if cacheShortCircuited {
+ return transition, nil
+ }
+
+ phase := transition.Info().Phase()
+ if phase.IsSuccess() {
+ status := corepb.CatalogCacheStatus_CACHE_POPULATED
+ if err := r.writeTaskOutputsToCache(ctx, tCtx, cacheCfg.key); err != nil {
+ if grpcstatus.Code(err) == codes.AlreadyExists {
+ status = corepb.CatalogCacheStatus_CACHE_POPULATED
+ } else {
+ log.FromContext(ctx).Error(err, "failed to write task outputs to cache")
+ status = corepb.CatalogCacheStatus_CACHE_PUT_FAILURE
+ }
+ }
+ appendCacheStatus(transition.Info().Info(), status)
+ if cacheCfg.serializable {
+ if err := r.releaseCacheReservation(ctx, cacheCfg); err != nil {
+ log.FromContext(ctx).Error(err, "failed to release cache reservation after success")
+ }
+ }
+ return transition, nil
+ }
+
+ if cacheCfg.serializable && (phase.IsFailure() || phase.IsAborted()) {
+ if err := r.releaseCacheReservation(ctx, cacheCfg); err != nil {
+ log.FromContext(ctx).Error(err, "failed to release cache reservation after terminal failure")
+ }
+ }
+
+ return transition, nil
+}
+
+func buildTaskCacheConfig(
+ ctx context.Context,
+ taskAction *flyteorgv1.TaskAction,
+ tCtx pluginsCore.TaskExecutionContext,
+) (*taskCacheConfig, bool, error) {
+ taskTemplate, err := tCtx.TaskReader().Read(ctx)
+ if err != nil {
+ return nil, false, fmt.Errorf("reading task template for cache handling: %w", err)
+ }
+
+ metadata := taskTemplate.GetMetadata()
+ if metadata == nil || !metadata.GetDiscoverable() {
+ return nil, false, nil
+ }
+ if taskTemplate.GetId() == nil {
+ return nil, false, fmt.Errorf("discoverable task is missing identifier")
+ }
+
+ key := catalog.Key{
+ Identifier: proto.Clone(taskTemplate.GetId()).(*corepb.Identifier),
+ CacheVersion: taskAction.Spec.CacheKey,
+ CacheIgnoreInputVars: metadata.GetCacheIgnoreInputVars(),
+ TypedInterface: taskTemplate.GetInterface(),
+ InputReader: tCtx.InputReader(),
+ }
+
+ return &taskCacheConfig{
+ key: key,
+ ownerID: cacheReservationOwnerID(taskAction),
+ serializable: metadata.GetCacheSerializable(),
+ }, true, nil
+}
+
+func cacheReservationOwnerID(taskAction *flyteorgv1.TaskAction) string {
+ return types.NamespacedName{Name: taskAction.Name, Namespace: taskAction.Namespace}.String()
+}
+
+func (r *TaskActionReconciler) writeTaskOutputsToCache(ctx context.Context, tCtx pluginsCore.TaskExecutionContext, key catalog.Key) error {
+ outputPaths := ioutils.NewReadOnlyOutputFilePaths(ctx, r.DataStore, tCtx.OutputWriter().GetOutputPrefixPath())
+ outputReader := ioutils.NewRemoteFileOutputReader(ctx, r.DataStore, outputPaths, 0)
+ _, err := r.Catalog.Put(ctx, key, outputReader, cacheMetadataForUpload(tCtx, key.Identifier))
+ return err
+}
+
+func (r *TaskActionReconciler) releaseCacheReservation(ctx context.Context, cacheCfg *taskCacheConfig) error {
+ if r.Catalog == nil || cacheCfg == nil || !cacheCfg.serializable {
+ return nil
+ }
+
+ return r.Catalog.ReleaseReservation(ctx, cacheCfg.key, cacheCfg.ownerID)
+}
+
+func cacheMetadataForUpload(tCtx pluginsCore.TaskExecutionContext, taskID *corepb.Identifier) catalog.Metadata {
+ taskExecID := proto.Clone(tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetID()).(*corepb.TaskExecutionIdentifier)
+ taskExecID.TaskId = proto.Clone(taskID).(*corepb.Identifier)
+
+ return catalog.Metadata{
+ WorkflowExecutionIdentifier: taskExecID.GetNodeExecutionId().GetExecutionId(),
+ NodeExecutionIdentifier: taskExecID.GetNodeExecutionId(),
+ TaskExecutionIdentifier: taskExecID,
+ CreatedAt: timestamppb.Now(),
+ }
+}
+
+func cacheTaskInfo(status corepb.CatalogCacheStatus, reason string) *pluginsCore.TaskInfo {
+ now := time.Now()
+ return &pluginsCore.TaskInfo{
+ OccurredAt: &now,
+ ExternalResources: []*pluginsCore.ExternalResource{
+ {
+ ExternalID: "cache",
+ CacheStatus: status,
+ },
+ },
+ AdditionalReasons: []pluginsCore.ReasonInfo{
+ {Reason: reason, OccurredAt: &now},
+ },
+ }
+}
+
+func appendCacheStatus(info *pluginsCore.TaskInfo, status corepb.CatalogCacheStatus) {
+ if info == nil {
+ return
+ }
+ info.ExternalResources = append(info.ExternalResources, &pluginsCore.ExternalResource{
+ ExternalID: "cache",
+ CacheStatus: status,
+ })
+}
diff --git a/executor/pkg/controller/taskaction_cache_test.go b/executor/pkg/controller/taskaction_cache_test.go
new file mode 100644
index 00000000000..9d1aabed7b4
--- /dev/null
+++ b/executor/pkg/controller/taskaction_cache_test.go
@@ -0,0 +1,301 @@
+package controller
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/grpc/codes"
+ grpcstatus "google.golang.org/grpc/status"
+ "google.golang.org/protobuf/proto"
+
+ flyteorgv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+ executorplugin "github.com/flyteorg/flyte/v2/executor/pkg/plugin"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/catalog"
+ pluginsCore "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/core"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/io"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/ioutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/cacheservice"
+ corepb "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type stubCatalogClient struct {
+ getFunc func(context.Context, catalog.Key) (catalog.Entry, error)
+ getOrExtendReservationFunc func(context.Context, catalog.Key, string, time.Duration) (*cacheservice.Reservation, error)
+ putFunc func(context.Context, catalog.Key, io.OutputReader, catalog.Metadata) (catalog.Status, error)
+ releaseReservationFunc func(context.Context, catalog.Key, string) error
+}
+
+func (s *stubCatalogClient) Get(ctx context.Context, key catalog.Key) (catalog.Entry, error) {
+ return s.getFunc(ctx, key)
+}
+
+func (s *stubCatalogClient) GetOrExtendReservation(ctx context.Context, key catalog.Key, ownerID string, heartbeatInterval time.Duration) (*cacheservice.Reservation, error) {
+ return s.getOrExtendReservationFunc(ctx, key, ownerID, heartbeatInterval)
+}
+
+func (s *stubCatalogClient) Put(ctx context.Context, key catalog.Key, reader io.OutputReader, metadata catalog.Metadata) (catalog.Status, error) {
+ return s.putFunc(ctx, key, reader, metadata)
+}
+
+func (s *stubCatalogClient) Update(context.Context, catalog.Key, io.OutputReader, catalog.Metadata) (catalog.Status, error) {
+ return catalog.Status{}, nil
+}
+
+func (s *stubCatalogClient) ReleaseReservation(ctx context.Context, key catalog.Key, ownerID string) error {
+ return s.releaseReservationFunc(ctx, key, ownerID)
+}
+
+func (s *stubCatalogClient) GetReservationCache(string) catalog.ReservationCache {
+ return catalog.ReservationCache{}
+}
+
+func (s *stubCatalogClient) UpdateReservationCache(string, catalog.ReservationCache) {}
+
+func TestBuildTaskCacheConfig(t *testing.T) {
+ ensureTestMetricKeys()
+ ctx := context.Background()
+ taskAction, dataStore := newCacheableTaskAction(t, true, true)
+ tCtx := newTaskExecutionContext(t, taskAction, dataStore)
+
+ cfg, ok, err := buildTaskCacheConfig(ctx, taskAction, tCtx)
+ require.NoError(t, err)
+ require.True(t, ok)
+ assert.Empty(t, cfg.key.CacheKey)
+ assert.Equal(t, taskAction.Spec.CacheKey, cfg.key.CacheVersion)
+ assert.Equal(t, []string{"ignored_input"}, cfg.key.CacheIgnoreInputVars)
+ assert.True(t, cfg.serializable)
+ assert.Equal(t, "default/cacheable-action", cfg.ownerID)
+}
+
+func TestHandleCacheBeforeExecutionHit(t *testing.T) {
+ ensureTestMetricKeys()
+ ctx := context.Background()
+ taskAction, dataStore := newCacheableTaskAction(t, true, false)
+ tCtx := newTaskExecutionContext(t, taskAction, dataStore)
+ expected := &corepb.LiteralMap{
+ Literals: map[string]*corepb.Literal{
+ "o0": {
+ Value: &corepb.Literal_Scalar{
+ Scalar: &corepb.Scalar{
+ Value: &corepb.Scalar_Primitive{
+ Primitive: &corepb.Primitive{Value: &corepb.Primitive_StringValue{StringValue: "cached"}},
+ },
+ },
+ },
+ },
+ },
+ }
+
+ r := &TaskActionReconciler{
+ DataStore: dataStore,
+ Catalog: &stubCatalogClient{
+ getFunc: func(context.Context, catalog.Key) (catalog.Entry, error) {
+ return catalog.NewCatalogEntry(ioutils.NewInMemoryOutputReader(expected, nil, nil), catalog.NewStatus(corepb.CatalogCacheStatus_CACHE_HIT, nil)), nil
+ },
+ },
+ }
+
+ transition, handled, err := r.evaluateCacheBeforeExecution(ctx, taskAction, tCtx)
+ require.NoError(t, err)
+ require.True(t, handled)
+ assert.Equal(t, pluginsCore.PhaseSuccess, transition.Info().Phase())
+ assert.Equal(t, corepb.CatalogCacheStatus_CACHE_HIT, transition.Info().Info().ExternalResources[0].CacheStatus)
+
+ actual := &corepb.LiteralMap{}
+ require.NoError(t, dataStore.ReadProtobuf(ctx, tCtx.OutputWriter().GetOutputPath(), actual))
+ assert.True(t, proto.Equal(expected, actual))
+}
+
+func TestHandleCacheBeforeExecutionWaitingForReservation(t *testing.T) {
+ ensureTestMetricKeys()
+ ctx := context.Background()
+ taskAction, dataStore := newCacheableTaskAction(t, true, true)
+ tCtx := newTaskExecutionContext(t, taskAction, dataStore)
+
+ r := &TaskActionReconciler{
+ DataStore: dataStore,
+ Catalog: &stubCatalogClient{
+ getFunc: func(context.Context, catalog.Key) (catalog.Entry, error) {
+ return catalog.Entry{}, grpcstatus.Error(codes.NotFound, "miss")
+ },
+ getOrExtendReservationFunc: func(_ context.Context, _ catalog.Key, ownerID string, heartbeat time.Duration) (*cacheservice.Reservation, error) {
+ assert.Equal(t, cacheReservationHeartbeatInterval, heartbeat)
+ assert.Equal(t, "default/cacheable-action", ownerID)
+ return &cacheservice.Reservation{OwnerId: "default/other-action"}, nil
+ },
+ },
+ }
+
+ transition, handled, err := r.evaluateCacheBeforeExecution(ctx, taskAction, tCtx)
+ require.NoError(t, err)
+ require.True(t, handled)
+ assert.Equal(t, pluginsCore.PhaseWaitingForCache, transition.Info().Phase())
+ assert.Equal(t, corepb.CatalogCacheStatus_CACHE_MISS, transition.Info().Info().ExternalResources[0].CacheStatus)
+}
+
+func TestHandleCacheBeforeExecutionMissWithoutSerializableSkipsReservation(t *testing.T) {
+ ensureTestMetricKeys()
+ ctx := context.Background()
+ taskAction, dataStore := newCacheableTaskAction(t, true, false)
+ tCtx := newTaskExecutionContext(t, taskAction, dataStore)
+
+ r := &TaskActionReconciler{
+ DataStore: dataStore,
+ Catalog: &stubCatalogClient{
+ getFunc: func(context.Context, catalog.Key) (catalog.Entry, error) {
+ return catalog.Entry{}, grpcstatus.Error(codes.NotFound, "miss")
+ },
+ getOrExtendReservationFunc: func(context.Context, catalog.Key, string, time.Duration) (*cacheservice.Reservation, error) {
+ t.Fatal("reservation should not be requested for non-serializable cache")
+ return nil, nil
+ },
+ },
+ }
+
+ transition, handled, err := r.evaluateCacheBeforeExecution(ctx, taskAction, tCtx)
+ require.NoError(t, err)
+ require.False(t, handled)
+ assert.Equal(t, pluginsCore.UnknownTransition, transition)
+}
+
+func TestHandleCacheAfterExecutionWritesBackAndReleasesReservation(t *testing.T) {
+ ensureTestMetricKeys()
+ ctx := context.Background()
+ taskAction, dataStore := newCacheableTaskAction(t, true, true)
+ tCtx := newTaskExecutionContext(t, taskAction, dataStore)
+ require.NoError(t, tCtx.OutputWriter().Put(ctx, ioutils.NewInMemoryOutputReader(&corepb.LiteralMap{}, nil, nil)))
+
+ putCalled := false
+ released := false
+ r := &TaskActionReconciler{
+ DataStore: dataStore,
+ Catalog: &stubCatalogClient{
+ getFunc: func(context.Context, catalog.Key) (catalog.Entry, error) {
+ return catalog.Entry{}, nil
+ },
+ putFunc: func(_ context.Context, key catalog.Key, reader io.OutputReader, metadata catalog.Metadata) (catalog.Status, error) {
+ putCalled = true
+ assert.Empty(t, key.CacheKey)
+ assert.Equal(t, taskAction.Spec.CacheKey, key.CacheVersion)
+ assert.Equal(t, "task-name", metadata.TaskExecutionIdentifier.GetTaskId().GetName())
+ return catalog.NewStatus(corepb.CatalogCacheStatus_CACHE_POPULATED, nil), nil
+ },
+ releaseReservationFunc: func(_ context.Context, _ catalog.Key, ownerID string) error {
+ released = true
+ assert.Equal(t, "default/cacheable-action", ownerID)
+ return nil
+ },
+ },
+ }
+
+ transition := pluginsCore.DoTransition(pluginsCore.PhaseInfoSuccess(&pluginsCore.TaskInfo{}))
+ transition, err := r.finalizeCacheAfterExecution(ctx, taskAction, tCtx, transition, false)
+ require.NoError(t, err)
+ assert.True(t, putCalled)
+ assert.True(t, released)
+ require.Len(t, transition.Info().Info().ExternalResources, 1)
+ assert.Equal(t, corepb.CatalogCacheStatus_CACHE_POPULATED, transition.Info().Info().ExternalResources[0].CacheStatus)
+}
+
+func TestHandleCacheAfterExecutionReleasesReservationOnFailure(t *testing.T) {
+ ensureTestMetricKeys()
+ ctx := context.Background()
+ taskAction, dataStore := newCacheableTaskAction(t, true, true)
+ tCtx := newTaskExecutionContext(t, taskAction, dataStore)
+
+ released := false
+ r := &TaskActionReconciler{
+ DataStore: dataStore,
+ Catalog: &stubCatalogClient{
+ releaseReservationFunc: func(_ context.Context, _ catalog.Key, ownerID string) error {
+ released = true
+ assert.Equal(t, "default/cacheable-action", ownerID)
+ return nil
+ },
+ },
+ }
+
+ transition := pluginsCore.DoTransition(pluginsCore.PhaseInfoFailure("User", "boom", &pluginsCore.TaskInfo{}))
+ _, err := r.finalizeCacheAfterExecution(ctx, taskAction, tCtx, transition, false)
+ require.NoError(t, err)
+ assert.True(t, released)
+}
+
+func newCacheableTaskAction(t *testing.T, discoverable bool, serializable bool) (*flyteorgv1.TaskAction, *storage.DataStore) {
+ t.Helper()
+
+ dataStore, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, promutils.NewTestScope())
+ require.NoError(t, err)
+
+ taskTemplate := &corepb.TaskTemplate{
+ Id: &corepb.Identifier{
+ ResourceType: corepb.ResourceType_TASK,
+ Project: "project",
+ Domain: "domain",
+ Name: "task-name",
+ Version: "task-version",
+ },
+ Type: "python-task",
+ Metadata: &corepb.TaskMetadata{
+ Discoverable: discoverable,
+ DiscoveryVersion: "discovery-v1",
+ CacheSerializable: serializable,
+ CacheIgnoreInputVars: []string{"ignored_input"},
+ Runtime: &corepb.RuntimeMetadata{
+ Type: corepb.RuntimeMetadata_FLYTE_SDK,
+ },
+ },
+ Interface: &corepb.TypedInterface{},
+ Target: &corepb.TaskTemplate_Container{
+ Container: &corepb.Container{
+ Image: "python:3.11",
+ Command: []string{"echo"},
+ },
+ },
+ }
+ taskTemplateBytes, err := proto.Marshal(taskTemplate)
+ require.NoError(t, err)
+
+ taskAction := &flyteorgv1.TaskAction{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "cacheable-action",
+ Namespace: "default",
+ },
+ Spec: flyteorgv1.TaskActionSpec{
+ RunName: "run-name",
+ Project: "project",
+ Domain: "domain",
+ ActionName: "action-name",
+ InputURI: "s3://bucket/inputs.pb",
+ RunOutputBase: "s3://bucket/run-output",
+ CacheKey: "precomputed-cache-key",
+ TaskType: "python-task",
+ TaskTemplate: taskTemplateBytes,
+ },
+ }
+ return taskAction, dataStore
+}
+
+func newTaskExecutionContext(t *testing.T, taskAction *flyteorgv1.TaskAction, dataStore *storage.DataStore) pluginsCore.TaskExecutionContext {
+ t.Helper()
+
+ err := dataStore.WriteProtobuf(context.Background(), storage.DataReference(taskAction.Spec.InputURI), storage.Options{}, &corepb.LiteralMap{})
+ require.NoError(t, err)
+
+ tCtx, err := executorplugin.NewTaskExecutionContext(
+ taskAction,
+ dataStore,
+ executorplugin.NewPluginStateManager(nil, 0),
+ nil,
+ nil,
+ nil,
+ )
+ require.NoError(t, err)
+ return tCtx
+}
diff --git a/executor/pkg/controller/taskaction_controller.go b/executor/pkg/controller/taskaction_controller.go
new file mode 100644
index 00000000000..d30cbd2f798
--- /dev/null
+++ b/executor/pkg/controller/taskaction_controller.go
@@ -0,0 +1,809 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "k8s.io/client-go/util/retry"
+ "reflect"
+ "strings"
+ "time"
+
+ "connectrpc.com/connect"
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/meta"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/client-go/tools/record"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+ "sigs.k8s.io/controller-runtime/pkg/log"
+
+ flyteorgv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+ "github.com/flyteorg/flyte/v2/executor/pkg/plugin"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/catalog"
+ pluginsCore "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/core"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common"
+ core "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+ task "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/task"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow/workflowconnect"
+ "google.golang.org/protobuf/types/known/timestamppb"
+)
+
+const (
+ TaskActionDefaultRequeueDuration = 10 * time.Second
+ taskActionFinalizer = "flyte.org/plugin-finalizer"
+
+ // LabelTerminationStatus marks a TaskAction as terminated for GC discovery.
+ LabelTerminationStatus = "flyte.org/termination-status"
+ // LabelCompletedTime records the UTC time (minute precision) when the TaskAction became terminal.
+ LabelCompletedTime = "flyte.org/completed-time"
+ // LabelValueTerminated is the value for LabelTerminationStatus.
+ LabelValueTerminated = "terminated"
+ // labelTimeFormat is the time format used for the completed-time label (lexicographically ordered, minute precision).
+ labelTimeFormat = "2006-01-02.15-04"
+)
+
+type K8sEventType string
+
+const (
+ FailedUnmarshal K8sEventType = "FailedUnmarshal"
+ FailedValidation K8sEventType = "FailedValidation"
+ FailedPluginResolve K8sEventType = "FailedPluginResolve"
+ FailedPluginHandle K8sEventType = "FailedPluginHandle"
+)
+
+// TaskActionReconciler reconciles a TaskAction object
+type TaskActionReconciler struct {
+ client.Client
+ Scheme *runtime.Scheme
+ Recorder record.EventRecorder
+ PluginRegistry *plugin.Registry
+ DataStore *storage.DataStore
+ SecretManager pluginsCore.SecretManager
+ ResourceManager pluginsCore.ResourceManager
+ CatalogClient catalog.AsyncClient
+ Catalog catalog.Client
+ eventsClient workflowconnect.EventsProxyServiceClient
+ cluster string
+}
+
+// NewTaskActionReconciler creates a new TaskActionReconciler
+func NewTaskActionReconciler(
+ c client.Client,
+ scheme *runtime.Scheme,
+ registry *plugin.Registry,
+ dataStore *storage.DataStore,
+ eventsClient workflowconnect.EventsProxyServiceClient,
+ cluster string,
+) *TaskActionReconciler {
+ return &TaskActionReconciler{
+ Client: c,
+ Scheme: scheme,
+ PluginRegistry: registry,
+ DataStore: dataStore,
+ eventsClient: eventsClient,
+ cluster: cluster,
+ }
+}
+
+// +kubebuilder:rbac:groups=flyte.org,resources=taskactions,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=flyte.org,resources=taskactions/status,verbs=get;update;patch
+// +kubebuilder:rbac:groups=flyte.org,resources=taskactions/finalizers,verbs=update
+// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete
+
+// Reconcile is part of the main kubernetes reconciliation loop which aims to
+// move the current state of the cluster closer to the desired state.
+func (r *TaskActionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+ logger := log.FromContext(ctx)
+
+ // Fetch the TaskAction instance
+ taskAction := &flyteorgv1.TaskAction{}
+ if err := r.Get(ctx, req.NamespacedName, taskAction); err != nil {
+ return ctrl.Result{}, client.IgnoreNotFound(err)
+ }
+
+ // Please do NOT modify `originalTaskActionInstance` in the following code. This is for checking
+ // if the TaskAction instance changes
+ originalTaskActionInstance := taskAction.DeepCopy()
+
+ // Handle deletion
+ if !taskAction.DeletionTimestamp.IsZero() {
+ return r.handleAbortAndFinalize(ctx, taskAction)
+ }
+
+ // Check terminal conditions -- short-circuit
+ if isTerminal(taskAction) {
+ if err := r.ensureTerminalLabels(ctx, taskAction); err != nil {
+ return ctrl.Result{}, err
+ }
+ return ctrl.Result{}, nil
+ }
+
+ // Validate spec fields and resolve plugin before adding the finalizer
+ // If either fails, the resource is marked terminal and not requeued — no finalizer to clean up
+ p, reason, err := validateTaskAction(taskAction, r.PluginRegistry)
+ if err != nil {
+ logger.Error(err, "TaskAction validation failed")
+ eventType := FailedValidation
+ if reason == flyteorgv1.ConditionReasonPluginNotFound {
+ eventType = FailedPluginResolve
+ }
+ r.Recorder.Eventf(taskAction, corev1.EventTypeWarning, string(eventType), "%v", err)
+ setCondition(taskAction, flyteorgv1.ConditionTypeFailed, metav1.ConditionTrue, reason, err.Error())
+ setCondition(taskAction, flyteorgv1.ConditionTypeProgressing, metav1.ConditionFalse, reason, err.Error())
+ _ = r.Status().Update(ctx, taskAction)
+ return ctrl.Result{}, nil // terminal — do not requeue
+ }
+
+ // Ensure finalizer is present (once validation passes)
+ if !controllerutil.ContainsFinalizer(taskAction, taskActionFinalizer) {
+ controllerutil.AddFinalizer(taskAction, taskActionFinalizer)
+ if err := r.Update(ctx, taskAction); err != nil {
+ logger.Error(err, "Failed to update TaskAction with finalizer")
+ return ctrl.Result{}, err
+ }
+ return ctrl.Result{}, nil
+ }
+
+ // Build PluginStateManager from persisted state
+ stateMgr := plugin.NewPluginStateManager(
+ taskAction.Status.PluginState,
+ taskAction.Status.PluginStateVersion,
+ )
+
+ // Build TaskExecutionContext
+ tCtx, err := plugin.NewTaskExecutionContext(
+ taskAction,
+ r.DataStore,
+ stateMgr,
+ r.SecretManager,
+ r.ResourceManager,
+ r.CatalogClient,
+ )
+ if err != nil {
+ logger.Error(err, "failed to build task execution context")
+ return ctrl.Result{RequeueAfter: TaskActionDefaultRequeueDuration}, nil
+ }
+
+ // cacheShortCircuited is true when cache handling already decided the outcome,
+ // either via cache hit or waiting on the reservation owner.
+ var cacheShortCircuited bool
+ transition, cacheShortCircuited, err := r.evaluateCacheBeforeExecution(ctx, taskAction, tCtx)
+ if err != nil {
+ logger.Error(err, "cache pre-execution handling failed")
+ return ctrl.Result{RequeueAfter: TaskActionDefaultRequeueDuration}, nil
+ }
+ // Even when cache handling short-circuits execution, we still continue through the
+ // shared reconcile tail below so the derived transition updates conditions, status,
+ // and emitted action events in the same way as the normal plugin path.
+
+ // Invoke plugin.Handle only when cache handling did not short-circuit execution.
+ if !cacheShortCircuited {
+ transition, err = p.Handle(ctx, tCtx)
+ if err != nil {
+ logger.Error(err, "plugin Handle failed", "plugin", p.GetID())
+ r.Recorder.Eventf(taskAction, corev1.EventTypeWarning, string(FailedPluginHandle),
+ "Plugin %q Handle failed: %v", p.GetID(), err)
+ return ctrl.Result{RequeueAfter: TaskActionDefaultRequeueDuration}, nil
+ }
+ }
+
+ if transition, err = r.finalizeCacheAfterExecution(ctx, taskAction, tCtx, transition, cacheShortCircuited); err != nil {
+ logger.Error(err, "cache post-execution handling failed")
+ return ctrl.Result{RequeueAfter: TaskActionDefaultRequeueDuration}, nil
+ }
+
+ // Map transition phase to TaskAction conditions
+ phaseInfo := transition.Info()
+
+ // In-place task restart: when a recoverable failure occurs, restart the pod within the
+ // same TaskAction rather than relying on the runs service to create a new TaskAction.
+ var restartAttempts uint32
+ if !cacheShortCircuited && phaseInfo.Phase() == pluginsCore.PhaseRetryableFailure {
+ currentAttempts := observedAttempts(taskAction)
+ maxAttempts := tCtx.TaskExecutionMetadata().GetMaxAttempts()
+
+ if currentAttempts < maxAttempts {
+ // Abort (delete) the current pod before incrementing attempts.
+ // tCtx was built with the current attempt number so Abort targets the right pod.
+ if abortErr := p.Abort(ctx, tCtx); abortErr != nil {
+ logger.Error(abortErr, "failed to abort pod during in-place restart")
+ }
+ // Track the new attempt count; applied to Status.Attempts after the stateMgr block.
+ restartAttempts = currentAttempts + 1
+ logger.Info("restarting task in-place", "attempt", currentAttempts+1, "maxAttempts", maxAttempts)
+ // Override the transition to Queued so the TaskAction stays non-terminal.
+ transition = pluginsCore.DoTransition(pluginsCore.PhaseInfoQueued(time.Now(), pluginsCore.DefaultPhaseVersion, "restarting task"))
+ phaseInfo = transition.Info()
+ } else {
+ // All retries exhausted — convert to a permanent (terminal) failure.
+ execErr := phaseInfo.Err()
+ if execErr == nil {
+ execErr = &core.ExecutionError{
+ Kind: core.ExecutionError_USER,
+ Code: "MaxRetriesExceeded",
+ Message: fmt.Sprintf("task failed after %d attempt(s)", currentAttempts),
+ }
+ }
+ transition = pluginsCore.DoTransition(pluginsCore.PhaseInfoFailed(pluginsCore.PhasePermanentFailure, execErr, phaseInfo.Info()))
+ phaseInfo = transition.Info()
+ }
+ }
+ mapPhaseToConditions(taskAction, phaseInfo)
+
+ // Update StateJSON for observability
+ actionSpec, _ := taskAction.Spec.GetActionSpec()
+ if actionSpec != nil {
+ taskAction.Status.StateJSON = createStateJSON(actionSpec, phaseInfo.Phase().String())
+ }
+
+ // Persist new PluginState
+ if newBytes, newVersion, written := stateMgr.GetNewState(); written {
+ taskAction.Status.PluginState = newBytes
+ taskAction.Status.PluginStateVersion = newVersion
+ }
+
+ // If an in-place restart was triggered, increment attempts and clear plugin state so the
+ // next reconcile starts fresh with PluginPhaseNotStarted and creates a new pod.
+ if restartAttempts > 0 {
+ taskAction.Status.Attempts = restartAttempts
+ taskAction.Status.PluginState = nil
+ taskAction.Status.PluginStateVersion = 0
+ }
+
+ taskAction.Status.PluginPhase = phaseInfo.Phase().String()
+ taskAction.Status.PluginPhaseVersion = phaseInfo.Version()
+ taskAction.Status.Attempts = observedAttempts(taskAction)
+ taskAction.Status.CacheStatus = observedCacheStatus(phaseInfo.Info())
+
+ if err := r.updateTaskActionStatus(ctx, originalTaskActionInstance, taskAction, phaseInfo); err != nil {
+ return ctrl.Result{}, err
+ }
+
+ // If the TaskAction just became terminal, stamp GC labels
+ if isTerminal(taskAction) {
+ if err := r.ensureTerminalLabels(ctx, taskAction); err != nil {
+ return ctrl.Result{}, err
+ }
+ }
+
+ return ctrl.Result{RequeueAfter: TaskActionDefaultRequeueDuration}, nil
+}
+
+// ensureTerminalLabels adds GC-related labels to a terminal TaskAction if not already present.
+// This is idempotent — if the labels are already set, it's a no-op.
+// Uses a MergeFrom patch instead of a full Update to reduce conflict surface with concurrent reconciles.
+func (r *TaskActionReconciler) ensureTerminalLabels(ctx context.Context, taskAction *flyteorgv1.TaskAction) error {
+ labels := taskAction.GetLabels()
+ if labels != nil && labels[LabelTerminationStatus] == LabelValueTerminated && labels[LabelCompletedTime] != "" {
+ return nil // already labeled
+ }
+
+ patch := client.MergeFrom(taskAction.DeepCopy())
+
+ if labels == nil {
+ labels = make(map[string]string)
+ }
+ labels[LabelTerminationStatus] = LabelValueTerminated
+ labels[LabelCompletedTime] = terminalTransitionTime(taskAction).Format(labelTimeFormat)
+ taskAction.SetLabels(labels)
+
+ if err := r.Patch(ctx, taskAction, patch); err != nil {
+ log.FromContext(ctx).Error(err, "failed to set terminal labels on TaskAction")
+ return err
+ }
+ return nil
+}
+
+// handleAbortAndFinalize handles the deletion of a TaskAction by aborting and finalizing the plugin.
+func (r *TaskActionReconciler) handleAbortAndFinalize(ctx context.Context, taskAction *flyteorgv1.TaskAction) (ctrl.Result, error) {
+ logger := log.FromContext(ctx)
+
+ if !controllerutil.ContainsFinalizer(taskAction, taskActionFinalizer) {
+ return ctrl.Result{}, nil
+ }
+
+ p, err := r.PluginRegistry.ResolvePlugin(taskAction.Spec.TaskType)
+ if err != nil {
+ logger.Info("Cannot resolve plugin for abort/finalize, removing finalizer", "error", err)
+ return r.removeFinalizer(ctx, taskAction)
+ }
+
+ stateMgr := plugin.NewPluginStateManager(
+ taskAction.Status.PluginState,
+ taskAction.Status.PluginStateVersion,
+ )
+
+ tCtx, err := plugin.NewTaskExecutionContext(
+ taskAction, r.DataStore, stateMgr, r.SecretManager, r.ResourceManager, r.CatalogClient,
+ )
+ if err != nil {
+ logger.Error(err, "failed to build context for abort/finalize")
+ r.Recorder.Eventf(taskAction, corev1.EventTypeWarning, "FinalizationSkipped",
+ "Could not build task execution context; skipping Abort/Finalize. Underlying resources may need manual cleanup: %v", err)
+ return r.removeFinalizer(ctx, taskAction)
+ }
+
+ if err := p.Abort(ctx, tCtx); err != nil {
+ logger.Error(err, "plugin Abort failed, will retry")
+ return ctrl.Result{RequeueAfter: TaskActionDefaultRequeueDuration}, nil
+ }
+
+ if err := p.Finalize(ctx, tCtx); err != nil {
+ logger.Error(err, "plugin Finalize failed, will retry")
+ return ctrl.Result{RequeueAfter: TaskActionDefaultRequeueDuration}, nil
+ }
+
+ if cacheCfg, ok, err := buildTaskCacheConfig(ctx, taskAction, tCtx); err != nil {
+ logger.Error(err, "failed to build cache config for finalization cleanup")
+ } else if ok {
+ if err := r.releaseCacheReservation(ctx, cacheCfg); err != nil {
+ logger.Error(err, "failed to release cache reservation during finalization cleanup")
+ }
+ }
+
+ abortTime := time.Now()
+ abortPhaseInfo := pluginsCore.PhaseInfoAborted(abortTime, pluginsCore.DefaultPhaseVersion, "aborted")
+ actionEvent := r.buildActionEvent(ctx, taskAction, abortPhaseInfo)
+ // buildActionEvent derives UpdatedTime from PhaseHistory, which doesn't include the
+ // abort transition. Override it so mergeEvents uses the actual abort time as end_time.
+ actionEvent.UpdatedTime = timestamppb.New(abortTime)
+ if _, err := r.eventsClient.Record(ctx, connect.NewRequest(&workflow.RecordRequest{
+ Events: []*workflow.ActionEvent{actionEvent},
+ })); err != nil {
+ logger.Error(err, "failed to emit abort event, will retry")
+ return ctrl.Result{RequeueAfter: TaskActionDefaultRequeueDuration}, nil
+ }
+
+ return r.removeFinalizer(ctx, taskAction)
+}
+
+func (r *TaskActionReconciler) removeFinalizer(ctx context.Context, taskAction *flyteorgv1.TaskAction) (ctrl.Result, error) {
+ controllerutil.RemoveFinalizer(taskAction, taskActionFinalizer)
+ if err := r.Update(ctx, taskAction); err != nil {
+ return ctrl.Result{}, err
+ }
+ return ctrl.Result{}, nil
+}
+
+// updateTaskActionStatus updates the TaskAction status only when the status has changed,
+// avoiding unnecessary API calls for unchanged state.
+func (r *TaskActionReconciler) updateTaskActionStatus(
+ ctx context.Context,
+ oldTaskAction, newTaskAction *flyteorgv1.TaskAction,
+ phaseInfo pluginsCore.PhaseInfo,
+) error {
+ logger := log.FromContext(ctx)
+
+ if !taskActionStatusChanged(oldTaskAction.Status, newTaskAction.Status) {
+ return nil
+ }
+
+ actionEvent := r.buildActionEvent(ctx, newTaskAction, phaseInfo)
+ if _, err := r.eventsClient.Record(ctx, connect.NewRequest(&workflow.RecordRequest{
+ Events: []*workflow.ActionEvent{actionEvent},
+ })); err != nil {
+ r.Recorder.Eventf(
+ newTaskAction,
+ corev1.EventTypeWarning,
+ "ActionEventPublishFailed",
+ "Failed to persist action event %q: %v",
+ actionEvent.GetId().GetName(),
+ err,
+ )
+ logger.Error(err, "failed to persist action event", "action", actionEvent.GetId().GetName())
+ return err
+ }
+
+ // The retry.RetryOnConflict will refetch the k8s resource to get the latest resource version
+ // This will resovle the conflict error caused by k8s optimistic lock when 2 reconcile loops updating the same CRD
+ if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+ latest := &flyteorgv1.TaskAction{}
+ if getErr := r.Get(ctx, client.ObjectKeyFromObject(newTaskAction), latest); getErr != nil {
+ return getErr
+ }
+ latest.Status = newTaskAction.Status
+ return r.Status().Update(ctx, latest)
+ }); err != nil {
+ logger.Error(err, "Error updating status", "name", oldTaskAction.Name, "error", err, "TaskAction", newTaskAction)
+ return err
+ }
+
+ return nil
+}
+
+func (r *TaskActionReconciler) buildActionEvent(
+ ctx context.Context,
+ taskAction *flyteorgv1.TaskAction,
+ phaseInfo pluginsCore.PhaseInfo,
+) *workflow.ActionEvent {
+ actionID := &common.ActionIdentifier{
+ Run: &common.RunIdentifier{
+ Project: taskAction.Spec.Project,
+ Domain: taskAction.Spec.Domain,
+ Name: taskAction.Spec.RunName,
+ },
+ Name: taskAction.Spec.ActionName,
+ }
+
+ info := phaseInfo.Info()
+ updatedTime := updatedTimestamp(taskAction.Status.PhaseHistory)
+
+ event := &workflow.ActionEvent{
+ Id: actionID,
+ Attempt: observedAttempts(taskAction),
+ Phase: phaseToActionPhase(phaseInfo.Phase()),
+ Version: phaseInfo.Version(),
+ UpdatedTime: updatedTime,
+ ErrorInfo: toActionErrorInfo(phaseInfo.Err()),
+ Cluster: r.cluster,
+ Outputs: outputRefs(ctx, taskAction),
+ ClusterEvents: toClusterEvents(phaseInfo, updatedTime),
+ ReportedTime: timestamppb.New(time.Now()),
+ }
+
+ if info != nil {
+ event.LogInfo = info.Logs
+ event.LogContext = info.LogContext
+ }
+ event.CacheStatus = observedCacheStatus(info)
+
+ return event
+}
+
+func observedAttempts(taskAction *flyteorgv1.TaskAction) uint32 {
+ if taskAction.Status.Attempts > 0 {
+ return taskAction.Status.Attempts
+ }
+ // if attempts is not set, default to 1
+ return 1
+}
+
+func observedCacheStatus(info *pluginsCore.TaskInfo) core.CatalogCacheStatus {
+ if info == nil {
+ return core.CatalogCacheStatus_CACHE_DISABLED
+ }
+ return cacheStatusFromExternalResources(info.ExternalResources)
+}
+
+func updatedTimestamp(history []flyteorgv1.PhaseTransition) *timestamppb.Timestamp {
+ if n := len(history); n > 0 {
+ return timestamppb.New(history[n-1].OccurredAt.Time)
+ }
+ return timestamppb.Now()
+}
+
+func outputRefs(ctx context.Context, taskAction *flyteorgv1.TaskAction) *task.OutputReferences {
+ if taskAction.Spec.RunOutputBase == "" {
+ return nil
+ }
+ attempt := observedAttempts(taskAction)
+ prefix, err := plugin.ComputeActionOutputPath(ctx, taskAction.Namespace, taskAction.Name, taskAction.Spec.RunOutputBase, taskAction.Spec.ActionName, attempt)
+ if err != nil {
+ return nil
+ }
+ base := strings.TrimRight(string(prefix), "/")
+ return &task.OutputReferences{
+ OutputUri: base + "/outputs.pb",
+ ReportUri: base + "/report.html",
+ }
+}
+
+func phaseToActionPhase(phase pluginsCore.Phase) common.ActionPhase {
+ switch phase {
+ case pluginsCore.PhaseNotReady, pluginsCore.PhaseQueued:
+ return common.ActionPhase_ACTION_PHASE_QUEUED
+ case pluginsCore.PhaseWaitingForResources, pluginsCore.PhaseWaitingForCache:
+ return common.ActionPhase_ACTION_PHASE_WAITING_FOR_RESOURCES
+ case pluginsCore.PhaseInitializing:
+ return common.ActionPhase_ACTION_PHASE_INITIALIZING
+ case pluginsCore.PhaseRunning:
+ return common.ActionPhase_ACTION_PHASE_RUNNING
+ case pluginsCore.PhaseSuccess:
+ return common.ActionPhase_ACTION_PHASE_SUCCEEDED
+ case pluginsCore.PhaseRetryableFailure, pluginsCore.PhasePermanentFailure:
+ return common.ActionPhase_ACTION_PHASE_FAILED
+ case pluginsCore.PhaseAborted:
+ return common.ActionPhase_ACTION_PHASE_ABORTED
+ default:
+ return common.ActionPhase_ACTION_PHASE_UNSPECIFIED
+ }
+}
+
+func toActionErrorInfo(err *core.ExecutionError) *workflow.ErrorInfo {
+ if err == nil {
+ return nil
+ }
+ out := &workflow.ErrorInfo{
+ Message: err.GetMessage(),
+ Kind: workflow.ErrorInfo_KIND_UNSPECIFIED,
+ }
+ switch err.GetKind() {
+ case core.ExecutionError_USER:
+ out.Kind = workflow.ErrorInfo_KIND_USER
+ case core.ExecutionError_SYSTEM:
+ out.Kind = workflow.ErrorInfo_KIND_SYSTEM
+ }
+ return out
+}
+
+func toClusterEvents(phaseInfo pluginsCore.PhaseInfo, fallbackTime *timestamppb.Timestamp) []*workflow.ClusterEvent {
+ info := phaseInfo.Info()
+ if phaseInfo.Reason() == "" && (info == nil || len(info.AdditionalReasons) == 0) {
+ return nil
+ }
+
+ out := []*workflow.ClusterEvent{}
+ if phaseInfo.Reason() != "" {
+ e := &workflow.ClusterEvent{
+ Message: phaseInfo.Reason(),
+ }
+ if info != nil && info.OccurredAt != nil {
+ e.OccurredAt = timestamppb.New(*info.OccurredAt)
+ } else {
+ e.OccurredAt = fallbackTime
+ }
+ out = append(out, e)
+ }
+
+ if info == nil {
+ return out
+ }
+
+ for _, reason := range info.AdditionalReasons {
+ e := &workflow.ClusterEvent{
+ Message: reason.Reason,
+ }
+ if reason.OccurredAt != nil {
+ e.OccurredAt = timestamppb.New(*reason.OccurredAt)
+ } else {
+ e.OccurredAt = fallbackTime
+ }
+ out = append(out, e)
+ }
+ return out
+}
+
+func cacheStatusFromExternalResources(resources []*pluginsCore.ExternalResource) core.CatalogCacheStatus {
+ for _, resource := range resources {
+ if resource == nil {
+ continue
+ }
+ // Return the first explicit cache status signal.
+ if resource.CacheStatus != core.CatalogCacheStatus_CACHE_DISABLED {
+ return resource.CacheStatus
+ }
+ }
+ return core.CatalogCacheStatus_CACHE_DISABLED
+}
+
+// taskActionStatusChanged reports whether any status field has changed between old and new,
+// covering plugin phase, state, state version, observability JSON, conditions, and phase history.
+func taskActionStatusChanged(oldStatus, newStatus flyteorgv1.TaskActionStatus) bool {
+ if oldStatus.StateJSON != newStatus.StateJSON ||
+ oldStatus.PluginStateVersion != newStatus.PluginStateVersion ||
+ oldStatus.PluginPhase != newStatus.PluginPhase ||
+ oldStatus.PluginPhaseVersion != newStatus.PluginPhaseVersion ||
+ oldStatus.Attempts != newStatus.Attempts ||
+ oldStatus.CacheStatus != newStatus.CacheStatus {
+ return true
+ }
+
+ if !bytes.Equal(oldStatus.PluginState, newStatus.PluginState) {
+ return true
+ }
+
+ if len(oldStatus.PhaseHistory) != len(newStatus.PhaseHistory) {
+ return true
+ }
+
+ return !reflect.DeepEqual(oldStatus.Conditions, newStatus.Conditions)
+}
+
+// mapPhaseToConditions maps a plugin PhaseInfo to TaskAction conditions.
+func mapPhaseToConditions(ta *flyteorgv1.TaskAction, info pluginsCore.PhaseInfo) {
+ var phaseName string
+ var msg string
+
+ switch info.Phase() {
+ case pluginsCore.PhaseNotReady, pluginsCore.PhaseQueued, pluginsCore.PhaseWaitingForResources, pluginsCore.PhaseWaitingForCache:
+ phaseName = string(flyteorgv1.ConditionReasonQueued)
+ msg = info.Reason()
+ setCondition(ta, flyteorgv1.ConditionTypeProgressing, metav1.ConditionTrue,
+ flyteorgv1.ConditionReasonQueued, msg)
+
+ case pluginsCore.PhaseInitializing:
+ phaseName = string(flyteorgv1.ConditionReasonInitializing)
+ msg = info.Reason()
+ setCondition(ta, flyteorgv1.ConditionTypeProgressing, metav1.ConditionTrue,
+ flyteorgv1.ConditionReasonInitializing, msg)
+
+ case pluginsCore.PhaseRunning:
+ phaseName = string(flyteorgv1.ConditionReasonExecuting)
+ msg = info.Reason()
+ setCondition(ta, flyteorgv1.ConditionTypeProgressing, metav1.ConditionTrue,
+ flyteorgv1.ConditionReasonExecuting, msg)
+
+ case pluginsCore.PhaseSuccess:
+ phaseName = string(flyteorgv1.ConditionReasonCompleted)
+ msg = "TaskAction completed successfully"
+ setCondition(ta, flyteorgv1.ConditionTypeProgressing, metav1.ConditionFalse,
+ flyteorgv1.ConditionReasonCompleted, "TaskAction has completed")
+ setCondition(ta, flyteorgv1.ConditionTypeSucceeded, metav1.ConditionTrue,
+ flyteorgv1.ConditionReasonCompleted, msg)
+
+ case pluginsCore.PhasePermanentFailure:
+ phaseName = string(flyteorgv1.ConditionReasonPermanentFailure)
+ msg = info.Reason()
+ if info.Err() != nil {
+ msg = info.Err().GetMessage()
+ }
+ setCondition(ta, flyteorgv1.ConditionTypeProgressing, metav1.ConditionFalse,
+ flyteorgv1.ConditionReasonPermanentFailure, msg)
+ setCondition(ta, flyteorgv1.ConditionTypeFailed, metav1.ConditionTrue,
+ flyteorgv1.ConditionReasonPermanentFailure, msg)
+
+ case pluginsCore.PhaseRetryableFailure:
+ phaseName = string(flyteorgv1.ConditionReasonRetryableFailure)
+ msg = info.Reason()
+ if info.Err() != nil {
+ msg = info.Err().GetMessage()
+ }
+ setCondition(ta, flyteorgv1.ConditionTypeProgressing, metav1.ConditionTrue,
+ flyteorgv1.ConditionReasonRetryableFailure, msg)
+
+ case pluginsCore.PhaseAborted:
+ phaseName = string(flyteorgv1.ConditionReasonAborted)
+ msg = "TaskAction was aborted"
+ setCondition(ta, flyteorgv1.ConditionTypeProgressing, metav1.ConditionFalse,
+ flyteorgv1.ConditionReasonAborted, msg)
+ setCondition(ta, flyteorgv1.ConditionTypeFailed, metav1.ConditionTrue,
+ flyteorgv1.ConditionReasonAborted, msg)
+ }
+
+ // Append to PhaseHistory if this is a new phase (dedup by checking last entry).
+ if phaseName != "" {
+ n := len(ta.Status.PhaseHistory)
+ if n == 0 || ta.Status.PhaseHistory[n-1].Phase != phaseName {
+ ta.Status.PhaseHistory = append(ta.Status.PhaseHistory, flyteorgv1.PhaseTransition{
+ Phase: phaseName,
+ OccurredAt: metav1.Now(),
+ Message: msg,
+ })
+ }
+ }
+}
+
+// isTerminal returns true if the TaskAction has reached a terminal condition.
+func isTerminal(ta *flyteorgv1.TaskAction) bool {
+ for _, cond := range ta.Status.Conditions {
+ if cond.Type == string(flyteorgv1.ConditionTypeSucceeded) && cond.Status == metav1.ConditionTrue {
+ return true
+ }
+ if cond.Type == string(flyteorgv1.ConditionTypeFailed) && cond.Status == metav1.ConditionTrue {
+ return true
+ }
+ }
+ return false
+}
+
+// terminalTransitionTime returns the LastTransitionTime from the terminal condition
+// (Succeeded or Failed). Falls back to time.Now().UTC() if no transition time is found.
+func terminalTransitionTime(ta *flyteorgv1.TaskAction) time.Time {
+ for _, cond := range ta.Status.Conditions {
+ if cond.Status != metav1.ConditionTrue {
+ continue
+ }
+ if cond.Type == string(flyteorgv1.ConditionTypeSucceeded) || cond.Type == string(flyteorgv1.ConditionTypeFailed) {
+ if !cond.LastTransitionTime.IsZero() {
+ return cond.LastTransitionTime.UTC()
+ }
+ break
+ }
+ }
+ return time.Now().UTC()
+}
+
+// createStateJSON creates a simplified state JSON for observability.
+func createStateJSON(actionSpec *workflow.ActionSpec, phase string) string {
+ state := map[string]interface{}{
+ "phase": phase,
+ "actionId": fmt.Sprintf("%s/%s", actionSpec.ActionId.Run.Name, actionSpec.ActionId.Name),
+ "timestamp": time.Now().Format(time.RFC3339),
+ }
+
+ stateBytes, err := json.Marshal(state)
+ if err != nil {
+ return "{}"
+ }
+ return string(stateBytes)
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *TaskActionReconciler) SetupWithManager(mgr ctrl.Manager) error {
+ return ctrl.NewControllerManagedBy(mgr).
+ For(&flyteorgv1.TaskAction{}).
+ Owns(&corev1.Pod{}).
+ Named("taskaction").
+ Complete(r)
+}
+
+// pluginResolver is satisfied by *plugin.Registry and allows mocking in tests.
+type pluginResolver interface {
+ ResolvePlugin(taskType string) (pluginsCore.Plugin, error)
+}
+
+// validateTaskAction checks that all required spec fields are populated and that a plugin
+// is registered for the given task type. Both checks happen before the finalizer is added,
+// so a failure here leaves the resource finalizer-free and trivially deletable.
+func validateTaskAction(taskAction *flyteorgv1.TaskAction, registry pluginResolver) (pluginsCore.Plugin, flyteorgv1.TaskActionConditionReason, error) {
+ var missing []string
+ if taskAction.Spec.RunName == "" {
+ missing = append(missing, "runName")
+ }
+ if taskAction.Spec.Project == "" {
+ missing = append(missing, "project")
+ }
+ if taskAction.Spec.Domain == "" {
+ missing = append(missing, "domain")
+ }
+ if taskAction.Spec.ActionName == "" {
+ missing = append(missing, "actionName")
+ }
+ if taskAction.Spec.TaskType == "" {
+ missing = append(missing, "taskType")
+ }
+ if len(taskAction.Spec.TaskTemplate) == 0 {
+ missing = append(missing, "taskTemplate")
+ }
+ if taskAction.Spec.InputURI == "" {
+ missing = append(missing, "inputUri")
+ }
+ if taskAction.Spec.RunOutputBase == "" {
+ missing = append(missing, "runOutputBase")
+ }
+ if len(missing) > 0 {
+ return nil, flyteorgv1.ConditionReasonInvalidSpec,
+ fmt.Errorf("required spec fields are empty: %v", missing)
+ }
+
+ p, err := registry.ResolvePlugin(taskAction.Spec.TaskType)
+ if err != nil {
+ return nil, flyteorgv1.ConditionReasonPluginNotFound,
+ fmt.Errorf("no plugin found for task type %q: %w", taskAction.Spec.TaskType, err)
+ }
+
+ return p, "", nil
+}
+
+// setCondition sets or updates a condition on the TaskAction.
+func setCondition(taskAction *flyteorgv1.TaskAction, conditionType flyteorgv1.TaskActionConditionType, status metav1.ConditionStatus, reason flyteorgv1.TaskActionConditionReason, message string) {
+ condition := metav1.Condition{
+ Type: string(conditionType),
+ Status: status,
+ Reason: string(reason),
+ Message: message,
+ }
+ meta.SetStatusCondition(&taskAction.Status.Conditions, condition)
+}
diff --git a/executor/pkg/controller/taskaction_controller_test.go b/executor/pkg/controller/taskaction_controller_test.go
new file mode 100644
index 00000000000..f30026cbd95
--- /dev/null
+++ b/executor/pkg/controller/taskaction_controller_test.go
@@ -0,0 +1,411 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+ "context"
+ "sync"
+ "time"
+
+ "connectrpc.com/connect"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/types/known/timestamppb"
+ "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/client-go/tools/record"
+ "sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+ flyteorgv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+ pluginsCore "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/core"
+ k8sPlugin "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/k8s"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow"
+)
+
+// fakeEventsClient is a no-op implementation of EventsProxyServiceClient for tests.
+type fakeEventsClient struct{}
+
+func (f *fakeEventsClient) Record(_ context.Context, _ *connect.Request[workflow.RecordRequest]) (*connect.Response[workflow.RecordResponse], error) {
+ return connect.NewResponse(&workflow.RecordResponse{}), nil
+}
+
+// recordingEventsClient captures all recorded ActionEvents for assertion in tests.
+type recordingEventsClient struct {
+ mu sync.Mutex
+ events []*workflow.ActionEvent
+}
+
+func (r *recordingEventsClient) Record(_ context.Context, req *connect.Request[workflow.RecordRequest]) (*connect.Response[workflow.RecordResponse], error) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.events = append(r.events, req.Msg.GetEvents()...)
+ return connect.NewResponse(&workflow.RecordResponse{}), nil
+}
+
+func (r *recordingEventsClient) RecordedEvents() []*workflow.ActionEvent {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ out := make([]*workflow.ActionEvent, len(r.events))
+ copy(out, r.events)
+ return out
+}
+
+// buildTaskTemplateBytes creates a minimal protobuf-serialized TaskTemplate
+// with a container spec that the pod plugin can use to build a Pod.
+func buildTaskTemplateBytes(taskType, image string) []byte {
+ tmpl := &core.TaskTemplate{
+ Type: taskType,
+ Target: &core.TaskTemplate_Container{
+ Container: &core.Container{
+ Image: image,
+ Command: []string{"echo"},
+ Args: []string{"hello"},
+ },
+ },
+ Metadata: &core.TaskMetadata{
+ Runtime: &core.RuntimeMetadata{
+ Type: core.RuntimeMetadata_FLYTE_SDK,
+ },
+ },
+ Interface: &core.TypedInterface{},
+ }
+ data, err := proto.Marshal(tmpl)
+ Expect(err).NotTo(HaveOccurred())
+ return data
+}
+
+// emptyPluginRegistry satisfies plugin.PluginRegistryIface with no registered plugins.
+type emptyPluginRegistry struct{}
+
+func (emptyPluginRegistry) GetCorePlugins() []pluginsCore.PluginEntry { return nil }
+func (emptyPluginRegistry) GetK8sPlugins() []k8sPlugin.PluginEntry { return nil }
+
+var _ = Describe("TaskAction Controller", func() {
+ Context("When reconciling a resource", func() {
+ const resourceName = "test-resource"
+
+ ctx := context.Background()
+
+ typeNamespacedName := types.NamespacedName{
+ Name: resourceName,
+ Namespace: "default",
+ }
+ taskaction := &flyteorgv1.TaskAction{}
+
+ BeforeEach(func() {
+ By("creating the custom resource for the Kind TaskAction")
+ err := k8sClient.Get(ctx, typeNamespacedName, taskaction)
+ if err != nil && errors.IsNotFound(err) {
+ resource := &flyteorgv1.TaskAction{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: resourceName,
+ Namespace: "default",
+ },
+ Spec: flyteorgv1.TaskActionSpec{
+ RunName: "test-run",
+ Project: "test-project",
+ Domain: "test-domain",
+ ActionName: "test-action",
+ InputURI: "/tmp/input",
+ RunOutputBase: "/tmp/output",
+ TaskType: "python-task",
+ TaskTemplate: buildTaskTemplateBytes("python-task", "python:3.11"),
+ },
+ }
+ Expect(k8sClient.Create(ctx, resource)).To(Succeed())
+ }
+ })
+
+ AfterEach(func() {
+ // TODO(user): Cleanup logic after each test, like removing the resource instance.
+ resource := &flyteorgv1.TaskAction{}
+ err := k8sClient.Get(ctx, typeNamespacedName, resource)
+ Expect(err).NotTo(HaveOccurred())
+
+ By("Cleanup the specific resource instance TaskAction")
+ Expect(k8sClient.Delete(ctx, resource)).To(Succeed())
+ })
+
+ It("should successfully reconcile the resource", func() {
+ By("Reconciling the created resource")
+
+ controllerReconciler := &TaskActionReconciler{
+ Client: k8sClient,
+ Scheme: k8sClient.Scheme(),
+ Recorder: record.NewFakeRecorder(10),
+ PluginRegistry: pluginRegistry,
+ DataStore: dataStore,
+ eventsClient: &fakeEventsClient{},
+ }
+
+ _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
+ NamespacedName: typeNamespacedName,
+ })
+ Expect(err).NotTo(HaveOccurred())
+
+ // After the first reconciliation the controller should have added
+ // a finalizer and/or set conditions on the TaskAction status.
+ updatedTaskAction := &flyteorgv1.TaskAction{}
+ err = k8sClient.Get(ctx, typeNamespacedName, updatedTaskAction)
+ Expect(err).NotTo(HaveOccurred())
+
+ // The first reconcile adds the finalizer; a second reconcile
+ // drives the plugin Handle path which sets conditions.
+ _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{
+ NamespacedName: typeNamespacedName,
+ })
+ Expect(err).NotTo(HaveOccurred())
+
+ err = k8sClient.Get(ctx, typeNamespacedName, updatedTaskAction)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(updatedTaskAction.Status.Conditions).NotTo(BeEmpty())
+ })
+ })
+
+ Context("When reconciling a terminal TaskAction", func() {
+ const terminalResourceName = "terminal-test-resource"
+
+ ctx := context.Background()
+
+ typeNamespacedName := types.NamespacedName{
+ Name: terminalResourceName,
+ Namespace: "default",
+ }
+
+ BeforeEach(func() {
+ By("creating a terminal TaskAction")
+ resource := &flyteorgv1.TaskAction{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: terminalResourceName,
+ Namespace: "default",
+ },
+ Spec: flyteorgv1.TaskActionSpec{
+ RunName: "test-run",
+ Project: "test-project",
+ Domain: "test-domain",
+ ActionName: "test-action",
+ InputURI: "/tmp/input",
+ RunOutputBase: "/tmp/output",
+ TaskType: "python-task",
+ TaskTemplate: buildTaskTemplateBytes("python-task", "python:3.11"),
+ },
+ }
+ Expect(k8sClient.Create(ctx, resource)).To(Succeed())
+
+ // Set terminal condition on status
+ resource.Status.Conditions = []metav1.Condition{
+ {
+ Type: string(flyteorgv1.ConditionTypeSucceeded),
+ Status: metav1.ConditionTrue,
+ Reason: string(flyteorgv1.ConditionReasonCompleted),
+ Message: "TaskAction completed successfully",
+ LastTransitionTime: metav1.Now(),
+ },
+ }
+ Expect(k8sClient.Status().Update(ctx, resource)).To(Succeed())
+ })
+
+ AfterEach(func() {
+ resource := &flyteorgv1.TaskAction{}
+ err := k8sClient.Get(ctx, typeNamespacedName, resource)
+ if err == nil {
+ Expect(k8sClient.Delete(ctx, resource)).To(Succeed())
+ }
+ })
+
+ It("should set GC labels on terminal TaskAction", func() {
+ By("Reconciling the terminal resource")
+
+ controllerReconciler := &TaskActionReconciler{
+ Client: k8sClient,
+ Scheme: k8sClient.Scheme(),
+ }
+
+ _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
+ NamespacedName: typeNamespacedName,
+ })
+ Expect(err).NotTo(HaveOccurred())
+
+ // Verify GC labels are set
+ updatedTaskAction := &flyteorgv1.TaskAction{}
+ err = k8sClient.Get(ctx, typeNamespacedName, updatedTaskAction)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(updatedTaskAction.GetLabels()).To(HaveKeyWithValue(LabelTerminationStatus, LabelValueTerminated))
+ Expect(updatedTaskAction.GetLabels()).To(HaveKey(LabelCompletedTime))
+ })
+ })
+
+ Context("mapPhaseToConditions", func() {
+ It("should keep PhaseHistory using controller time, not pod time", func() {
+ ta := &flyteorgv1.TaskAction{}
+ podTime := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) // far in the past
+ info := pluginsCore.PhaseInfoRunning(0, &pluginsCore.TaskInfo{
+ OccurredAt: &podTime,
+ })
+ before := time.Now().Add(-time.Second)
+ mapPhaseToConditions(ta, info)
+ after := time.Now().Add(time.Second)
+
+ Expect(ta.Status.PhaseHistory).To(HaveLen(1))
+ phTime := ta.Status.PhaseHistory[0].OccurredAt.Time
+ Expect(phTime.After(before)).To(BeTrue(), "PhaseHistory should use controller time, not pod time")
+ Expect(phTime.Before(after)).To(BeTrue(), "PhaseHistory should use controller time, not pod time")
+ })
+ })
+
+ Context("taskActionStatusChanged", func() {
+ It("should detect PhaseHistory changes", func() {
+ oldStatus := flyteorgv1.TaskActionStatus{
+ PhaseHistory: []flyteorgv1.PhaseTransition{
+ {Phase: "Queued", OccurredAt: metav1.Now()},
+ },
+ }
+ newStatus := flyteorgv1.TaskActionStatus{
+ PhaseHistory: []flyteorgv1.PhaseTransition{
+ {Phase: "Queued", OccurredAt: metav1.Now()},
+ {Phase: "Executing", OccurredAt: metav1.Now()},
+ },
+ }
+ Expect(taskActionStatusChanged(oldStatus, newStatus)).To(BeTrue())
+ })
+
+ It("should return false when nothing changed", func() {
+ now := metav1.Now()
+ status := flyteorgv1.TaskActionStatus{
+ PhaseHistory: []flyteorgv1.PhaseTransition{
+ {Phase: "Queued", OccurredAt: now},
+ },
+ }
+ Expect(taskActionStatusChanged(status, status)).To(BeFalse())
+ })
+
+ It("should detect PhaseHistory addition from empty", func() {
+ oldStatus := flyteorgv1.TaskActionStatus{}
+ newStatus := flyteorgv1.TaskActionStatus{
+ PhaseHistory: []flyteorgv1.PhaseTransition{
+ {Phase: "Queued", OccurredAt: metav1.Now()},
+ },
+ }
+ Expect(taskActionStatusChanged(oldStatus, newStatus)).To(BeTrue())
+ })
+ })
+
+ Context("When a TaskAction is deleted (abort flow)", func() {
+ const abortResourceName = "abort-test-resource"
+
+ ctx := context.Background()
+
+ typeNamespacedName := types.NamespacedName{
+ Name: abortResourceName,
+ Namespace: "default",
+ }
+
+ BeforeEach(func() {
+ resource := &flyteorgv1.TaskAction{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: abortResourceName,
+ Namespace: "default",
+ Finalizers: []string{taskActionFinalizer},
+ },
+ Spec: flyteorgv1.TaskActionSpec{
+ RunName: "abort-run",
+ Project: "abort-project",
+ Domain: "abort-domain",
+ ActionName: "abort-action",
+ InputURI: "/tmp/input",
+ RunOutputBase: "/tmp/output",
+ TaskType: "python-task",
+ TaskTemplate: buildTaskTemplateBytes("python-task", "python:3.11"),
+ },
+ }
+ Expect(k8sClient.Create(ctx, resource)).To(Succeed())
+ Expect(k8sClient.Delete(ctx, resource)).To(Succeed())
+ })
+
+ AfterEach(func() {
+ resource := &flyteorgv1.TaskAction{}
+ err := k8sClient.Get(ctx, typeNamespacedName, resource)
+ if err == nil {
+ resource.Finalizers = nil
+ Expect(k8sClient.Update(ctx, resource)).To(Succeed())
+ }
+ })
+
+ It("should emit an ACTION_PHASE_ABORTED event before removing the finalizer", func() {
+ recorder := &recordingEventsClient{}
+ reconciler := &TaskActionReconciler{
+ Client: k8sClient,
+ Scheme: k8sClient.Scheme(),
+ Recorder: record.NewFakeRecorder(10),
+ PluginRegistry: pluginRegistry,
+ DataStore: dataStore,
+ eventsClient: recorder,
+ }
+
+ result, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: typeNamespacedName})
+ Expect(err).NotTo(HaveOccurred())
+ Expect(result.RequeueAfter).To(BeZero())
+
+ // Finalizer should have been removed — object is gone.
+ deleted := &flyteorgv1.TaskAction{}
+ Expect(k8sClient.Get(ctx, typeNamespacedName, deleted)).NotTo(Succeed())
+
+ // An ABORTED event must have been emitted.
+ recorded := recorder.RecordedEvents()
+ Expect(recorded).NotTo(BeEmpty())
+ phases := make([]interface{}, len(recorded))
+ for i, e := range recorded {
+ phases[i] = e.GetPhase()
+ }
+ Expect(phases).To(ContainElement(common.ActionPhase_ACTION_PHASE_ABORTED))
+ })
+ })
+
+ Context("toClusterEvents", func() {
+ It("should include both phase reason and additional reasons", func() {
+ phaseOccurredAt := time.Date(2026, 4, 2, 10, 0, 0, 0, time.UTC)
+ eventOccurredAt := phaseOccurredAt.Add(2 * time.Minute)
+ fallbackTime := metav1.NewTime(phaseOccurredAt.Add(5 * time.Minute))
+
+ phaseInfo := pluginsCore.PhaseInfoQueuedWithTaskInfo(
+ phaseOccurredAt,
+ pluginsCore.DefaultPhaseVersion,
+ "cluster is creating",
+ &pluginsCore.TaskInfo{
+ OccurredAt: &phaseOccurredAt,
+ AdditionalReasons: []pluginsCore.ReasonInfo{
+ {
+ Reason: "Head pod pending",
+ OccurredAt: &eventOccurredAt,
+ },
+ },
+ },
+ )
+
+ events := toClusterEvents(phaseInfo, timestamppb.New(fallbackTime.Time))
+ Expect(events).To(HaveLen(2))
+ Expect(events[0].GetMessage()).To(Equal("cluster is creating"))
+ Expect(events[0].GetOccurredAt().AsTime()).To(Equal(phaseOccurredAt))
+ Expect(events[1].GetMessage()).To(Equal("Head pod pending"))
+ Expect(events[1].GetOccurredAt().AsTime()).To(Equal(eventOccurredAt))
+ })
+ })
+})
diff --git a/executor/pkg/controller/taskaction_validation_test.go b/executor/pkg/controller/taskaction_validation_test.go
new file mode 100644
index 00000000000..7b85f4ea27c
--- /dev/null
+++ b/executor/pkg/controller/taskaction_validation_test.go
@@ -0,0 +1,111 @@
+package controller
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "testing"
+
+ flyteorgv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+ pluginsCore "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/core"
+)
+
+// mockPluginResolver is a test double for pluginResolver.
+type mockPluginResolver struct {
+ plugin pluginsCore.Plugin
+ err error
+}
+
+func (m *mockPluginResolver) ResolvePlugin(_ string) (pluginsCore.Plugin, error) {
+ return m.plugin, m.err
+}
+
+// mockPlugin is a minimal pluginsCore.Plugin implementation for tests.
+type mockPlugin struct{}
+
+func (mockPlugin) GetID() string { return "mock" }
+func (mockPlugin) GetProperties() pluginsCore.PluginProperties { return pluginsCore.PluginProperties{} }
+func (mockPlugin) Handle(_ context.Context, _ pluginsCore.TaskExecutionContext) (pluginsCore.Transition, error) {
+ return pluginsCore.Transition{}, nil
+}
+func (mockPlugin) Abort(_ context.Context, _ pluginsCore.TaskExecutionContext) error { return nil }
+func (mockPlugin) Finalize(_ context.Context, _ pluginsCore.TaskExecutionContext) error { return nil }
+
+func validTaskAction() *flyteorgv1.TaskAction {
+ return &flyteorgv1.TaskAction{
+ Spec: flyteorgv1.TaskActionSpec{
+ RunName: "my-run",
+ Project: "my-project",
+ Domain: "my-domain",
+ ActionName: "my-action",
+ TaskType: "container",
+ TaskTemplate: []byte(`{}`),
+ InputURI: "s3://bucket/input",
+ RunOutputBase: "s3://bucket/output",
+ },
+ }
+}
+
+func TestValidateTaskAction_ValidSpec(t *testing.T) {
+ resolver := &mockPluginResolver{plugin: mockPlugin{}}
+ p, reason, err := validateTaskAction(validTaskAction(), resolver)
+ if err != nil {
+ t.Fatalf("expected no error, got: %v", err)
+ }
+ if reason != "" {
+ t.Fatalf("expected empty reason, got: %q", reason)
+ }
+ if p == nil {
+ t.Fatal("expected non-nil plugin")
+ }
+}
+
+func TestValidateTaskAction_MissingFields(t *testing.T) {
+ cases := []struct {
+ name string
+ mutate func(*flyteorgv1.TaskAction)
+ expectedField string
+ }{
+ {"missing runName", func(ta *flyteorgv1.TaskAction) { ta.Spec.RunName = "" }, "runName"},
+ {"missing project", func(ta *flyteorgv1.TaskAction) { ta.Spec.Project = "" }, "project"},
+ {"missing domain", func(ta *flyteorgv1.TaskAction) { ta.Spec.Domain = "" }, "domain"},
+ {"missing actionName", func(ta *flyteorgv1.TaskAction) { ta.Spec.ActionName = "" }, "actionName"},
+ {"missing taskType", func(ta *flyteorgv1.TaskAction) { ta.Spec.TaskType = "" }, "taskType"},
+ {"missing taskTemplate", func(ta *flyteorgv1.TaskAction) { ta.Spec.TaskTemplate = nil }, "taskTemplate"},
+ {"missing inputUri", func(ta *flyteorgv1.TaskAction) { ta.Spec.InputURI = "" }, "inputUri"},
+ {"missing runOutputBase", func(ta *flyteorgv1.TaskAction) { ta.Spec.RunOutputBase = "" }, "runOutputBase"},
+ }
+
+ resolver := &mockPluginResolver{plugin: mockPlugin{}}
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ ta := validTaskAction()
+ tc.mutate(ta)
+ _, reason, err := validateTaskAction(ta, resolver)
+ if err == nil {
+ t.Fatal("expected error, got nil")
+ }
+ if !strings.Contains(err.Error(), tc.expectedField) {
+ t.Errorf("expected error to mention %q, got: %v", tc.expectedField, err)
+ }
+ if reason != flyteorgv1.ConditionReasonInvalidSpec {
+ t.Errorf("expected reason %q, got %q", flyteorgv1.ConditionReasonInvalidSpec, reason)
+ }
+ })
+ }
+}
+
+func TestValidateTaskAction_PluginNotFound(t *testing.T) {
+ resolver := &mockPluginResolver{
+ plugin: nil,
+ err: fmt.Errorf("no plugin registered for task type %q", "container"),
+ }
+ _, reason, err := validateTaskAction(validTaskAction(), resolver)
+ if err == nil {
+ t.Fatal("expected error, got nil")
+ }
+ if reason != flyteorgv1.ConditionReasonPluginNotFound {
+ t.Errorf("expected reason %q, got %q", flyteorgv1.ConditionReasonPluginNotFound, reason)
+ }
+}
diff --git a/executor/pkg/controller/test_metrics_test.go b/executor/pkg/controller/test_metrics_test.go
new file mode 100644
index 00000000000..1447ac09601
--- /dev/null
+++ b/executor/pkg/controller/test_metrics_test.go
@@ -0,0 +1,21 @@
+package controller
+
+import (
+ "sync"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/contextutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils/labeled"
+)
+
+var metricsKeysOnce sync.Once
+
+func ensureTestMetricKeys() {
+ metricsKeysOnce.Do(func() {
+ labeled.SetMetricKeys(
+ contextutils.ProjectKey,
+ contextutils.DomainKey,
+ contextutils.WorkflowIDKey,
+ contextutils.TaskIDKey,
+ )
+ })
+}
diff --git a/executor/pkg/plugin/compute_action_output_path_test.go b/executor/pkg/plugin/compute_action_output_path_test.go
new file mode 100644
index 00000000000..aa8fe23898d
--- /dev/null
+++ b/executor/pkg/plugin/compute_action_output_path_test.go
@@ -0,0 +1,115 @@
+package plugin
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestComputeActionOutputPath(t *testing.T) {
+ ctx := context.Background()
+
+ t.Run("shard is inserted after bucket not after full base path", func(t *testing.T) {
+ path, err := ComputeActionOutputPath(ctx, "flyte", "my-action-abc", "s3://my-bucket/org/proj/dev/run123", "action-0", 1)
+ require.NoError(t, err)
+
+ u := string(path)
+ // shard must come before org/proj/dev, not after run123
+ shardIdx := strings.Index(u, "s3://my-bucket/") + len("s3://my-bucket/")
+ rest := u[shardIdx:]
+ parts := strings.SplitN(rest, "/", 2)
+ shard := parts[0]
+ assert.Len(t, shard, 2, "shard should be a 2-char prefix")
+ assert.True(t, strings.HasPrefix(u, "s3://my-bucket/"+shard+"/org/proj/dev/run123/"),
+ "shard %q should appear right after the bucket, got: %s", shard, u)
+ })
+
+ t.Run("attempt is the last path segment", func(t *testing.T) {
+ path, err := ComputeActionOutputPath(ctx, "flyte", "my-action-abc", "s3://my-bucket/org/proj/dev/run123", "action-0", 3)
+ require.NoError(t, err)
+
+ parts := strings.Split(strings.TrimRight(string(path), "/"), "/")
+ assert.Equal(t, "3", parts[len(parts)-1])
+ })
+
+ t.Run("action name is the second-to-last path segment", func(t *testing.T) {
+ path, err := ComputeActionOutputPath(ctx, "flyte", "my-action-abc", "s3://my-bucket/org/proj/dev/run123", "action-0", 1)
+ require.NoError(t, err)
+
+ parts := strings.Split(strings.TrimRight(string(path), "/"), "/")
+ assert.Equal(t, "action-0", parts[len(parts)-2])
+ })
+
+ t.Run("trailing slash on RunOutputBase is handled", func(t *testing.T) {
+ withSlash, err := ComputeActionOutputPath(ctx, "flyte", "my-action-abc", "s3://my-bucket/org/proj/dev/run123/", "action-0", 1)
+ require.NoError(t, err)
+ withoutSlash, err := ComputeActionOutputPath(ctx, "flyte", "my-action-abc", "s3://my-bucket/org/proj/dev/run123", "action-0", 1)
+ require.NoError(t, err)
+
+ assert.Equal(t, string(withoutSlash), string(withSlash))
+ })
+
+ t.Run("same namespace and name always produces the same shard", func(t *testing.T) {
+ p1, err := ComputeActionOutputPath(ctx, "flyte", "my-action-abc", "s3://my-bucket/run", "action-0", 1)
+ require.NoError(t, err)
+ p2, err := ComputeActionOutputPath(ctx, "flyte", "my-action-abc", "s3://my-bucket/run", "action-0", 1)
+ require.NoError(t, err)
+
+ assert.Equal(t, p1, p2)
+ })
+
+ t.Run("different namespace or name produces different shards", func(t *testing.T) {
+ // Collect shards from a handful of distinct namespace/name pairs and
+ // verify we see more than one unique value — confirming distribution.
+ inputs := [][2]string{
+ {"ns-a", "action-1"},
+ {"ns-b", "action-1"},
+ {"ns-a", "action-2"},
+ {"ns-c", "action-99"},
+ {"prod", "long-running-task-xyz"},
+ }
+ shards := make(map[string]struct{})
+ for _, in := range inputs {
+ p, err := ComputeActionOutputPath(ctx, in[0], in[1], "s3://my-bucket/run", "action-0", 1)
+ require.NoError(t, err)
+ // extract shard: segment right after bucket
+ u := string(p)
+ after := strings.TrimPrefix(u, "s3://my-bucket/")
+ shard := strings.SplitN(after, "/", 2)[0]
+ shards[shard] = struct{}{}
+ }
+ assert.Greater(t, len(shards), 1, "expected multiple distinct shards across different namespace/name pairs")
+ })
+
+ t.Run("different attempts produce different paths", func(t *testing.T) {
+ p1, err := ComputeActionOutputPath(ctx, "flyte", "my-action-abc", "s3://my-bucket/run", "action-0", 1)
+ require.NoError(t, err)
+ p2, err := ComputeActionOutputPath(ctx, "flyte", "my-action-abc", "s3://my-bucket/run", "action-0", 2)
+ require.NoError(t, err)
+
+ assert.NotEqual(t, p1, p2)
+ })
+
+ t.Run("shard does not change across attempts for the same action", func(t *testing.T) {
+ extractShard := func(path string) string {
+ after := strings.TrimPrefix(path, "s3://my-bucket/")
+ return strings.SplitN(after, "/", 2)[0]
+ }
+
+ p1, err := ComputeActionOutputPath(ctx, "flyte", "my-action-abc", "s3://my-bucket/run", "action-0", 1)
+ require.NoError(t, err)
+ p2, err := ComputeActionOutputPath(ctx, "flyte", "my-action-abc", "s3://my-bucket/run", "action-0", 2)
+ require.NoError(t, err)
+
+ assert.Equal(t, extractShard(string(p1)), extractShard(string(p2)),
+ "shard should be stable across retries since it only depends on namespace/name")
+ })
+
+ t.Run("invalid RunOutputBase returns error", func(t *testing.T) {
+ _, err := ComputeActionOutputPath(ctx, "flyte", "my-action-abc", "://bad url", "action-0", 1)
+ assert.Error(t, err)
+ })
+}
\ No newline at end of file
diff --git a/executor/pkg/plugin/k8s/event_watcher.go b/executor/pkg/plugin/k8s/event_watcher.go
new file mode 100644
index 00000000000..91cde79a016
--- /dev/null
+++ b/executor/pkg/plugin/k8s/event_watcher.go
@@ -0,0 +1,155 @@
+package k8s
+
+import (
+ "context"
+ "sort"
+ "sync"
+ "time"
+
+ eventsv1 "k8s.io/api/events/v1"
+ k8stypes "k8s.io/apimachinery/pkg/types"
+ toolscache "k8s.io/client-go/tools/cache"
+ ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache"
+)
+
+type watchedObjectKey struct {
+ Namespace string
+ Name string
+ Kind string
+}
+
+type eventInfo struct {
+ Message string
+ CreatedAt time.Time
+ RecordedAt time.Time
+ Reason string
+}
+
+type objectEventWatcher interface {
+ List(objectKey watchedObjectKey, createdAfter time.Time, recordedAfter time.Time) []*eventInfo
+}
+
+type controllerRuntimeEventWatcher struct {
+ objectCache sync.Map
+}
+
+type eventObjects struct {
+ mu sync.RWMutex
+ eventInfos map[k8stypes.NamespacedName]*eventInfo
+}
+
+func newControllerRuntimeEventWatcher(ctx context.Context, cache ctrlcache.Cache) (*controllerRuntimeEventWatcher, error) {
+ informer, err := cache.GetInformer(ctx, &eventsv1.Event{})
+ if err != nil {
+ return nil, err
+ }
+
+ watcher := &controllerRuntimeEventWatcher{}
+ if _, err := informer.AddEventHandler(watcher); err != nil {
+ return nil, err
+ }
+
+ return watcher, nil
+}
+
+func (w *controllerRuntimeEventWatcher) OnAdd(obj interface{}, _ bool) {
+ event, ok := obj.(*eventsv1.Event)
+ if !ok || event == nil {
+ return
+ }
+
+ objectKey := watchedObjectKey{
+ Namespace: event.Regarding.Namespace,
+ Name: event.Regarding.Name,
+ Kind: event.Regarding.Kind,
+ }
+ if objectKey.Name == "" || objectKey.Kind == "" {
+ return
+ }
+
+ eventKey := k8stypes.NamespacedName{Namespace: event.Namespace, Name: event.Name}
+
+ value, _ := w.objectCache.LoadOrStore(objectKey, &eventObjects{
+ eventInfos: make(map[k8stypes.NamespacedName]*eventInfo),
+ })
+ eventInfos := value.(*eventObjects)
+
+ eventInfos.mu.Lock()
+ eventInfos.eventInfos[eventKey] = &eventInfo{
+ Message: event.Note,
+ CreatedAt: event.CreationTimestamp.Time,
+ RecordedAt: time.Now(),
+ Reason: event.Reason,
+ }
+ eventInfos.mu.Unlock()
+}
+
+func (w *controllerRuntimeEventWatcher) OnUpdate(_, _ interface{}) {
+ // Ignore updates; we only need newly observed object events.
+}
+
+func (w *controllerRuntimeEventWatcher) OnDelete(obj interface{}) {
+ event, ok := obj.(*eventsv1.Event)
+ if !ok {
+ tombstone, ok := obj.(toolscache.DeletedFinalStateUnknown)
+ if !ok {
+ return
+ }
+ event, ok = tombstone.Obj.(*eventsv1.Event)
+ if !ok {
+ return
+ }
+ }
+
+ objectKey := watchedObjectKey{
+ Namespace: event.Regarding.Namespace,
+ Name: event.Regarding.Name,
+ Kind: event.Regarding.Kind,
+ }
+ if objectKey.Name == "" || objectKey.Kind == "" {
+ return
+ }
+
+ eventKey := k8stypes.NamespacedName{Namespace: event.Namespace, Name: event.Name}
+
+ value, ok := w.objectCache.Load(objectKey)
+ if !ok {
+ return
+ }
+ eventInfos := value.(*eventObjects)
+
+ eventInfos.mu.Lock()
+ defer eventInfos.mu.Unlock()
+
+ delete(eventInfos.eventInfos, eventKey)
+ // We intentionally do not delete empty buckets from objectCache. This avoids races where
+ // a new event is being added to the bucket while the top-level map entry is concurrently removed.
+}
+
+func (w *controllerRuntimeEventWatcher) List(objectKey watchedObjectKey, createdAfter time.Time, recordedAfter time.Time) []*eventInfo {
+ value, ok := w.objectCache.Load(objectKey)
+ if !ok {
+ return nil
+ }
+ eventInfos := value.(*eventObjects)
+
+ eventInfos.mu.RLock()
+ defer eventInfos.mu.RUnlock()
+
+ events := make([]*eventInfo, 0, len(eventInfos.eventInfos))
+ for _, info := range eventInfos.eventInfos {
+ if info.CreatedAt.After(createdAfter) ||
+ (info.CreatedAt.Equal(createdAfter) && info.RecordedAt.After(recordedAfter)) {
+ events = append(events, info)
+ }
+ }
+
+ sort.SliceStable(events, func(i, j int) bool {
+ if events[i].CreatedAt.Equal(events[j].CreatedAt) {
+ return events[i].RecordedAt.Before(events[j].RecordedAt)
+ }
+ return events[i].CreatedAt.Before(events[j].CreatedAt)
+ })
+
+ return events
+}
diff --git a/executor/pkg/plugin/k8s/plugin_context.go b/executor/pkg/plugin/k8s/plugin_context.go
new file mode 100644
index 00000000000..44291f5bc66
--- /dev/null
+++ b/executor/pkg/plugin/k8s/plugin_context.go
@@ -0,0 +1,71 @@
+package k8s
+
+import (
+ "context"
+ "fmt"
+
+ "sigs.k8s.io/controller-runtime/pkg/client"
+
+ pluginsCore "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/core"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/io"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/ioutils"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/k8s"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+)
+
+var _ k8s.PluginContext = &pluginContext{}
+
+// pluginContext adapts a TaskExecutionContext to k8s.PluginContext, adding the K8sReader
+// and a buffered OutputWriter that the k8s plugin ecosystem expects.
+type pluginContext struct {
+ pluginsCore.TaskExecutionContext
+ ow *ioutils.BufferedOutputWriter
+ k8sPluginState *k8s.PluginState
+ k8sReader client.Reader
+}
+
+func (p *pluginContext) OutputWriter() io.OutputWriter {
+ buf := ioutils.NewBufferedOutputWriter(context.TODO(), p.TaskExecutionContext.OutputWriter())
+ p.ow = buf
+ return buf
+}
+
+func (p *pluginContext) PluginStateReader() pluginsCore.PluginStateReader {
+ return &pluginStateReader{k8sPluginState: p.k8sPluginState}
+}
+
+func (p *pluginContext) K8sReader() client.Reader {
+ return p.k8sReader
+}
+
+func (p *pluginContext) DataStore() *storage.DataStore {
+ return p.TaskExecutionContext.DataStore()
+}
+
+func newPluginContext(tCtx pluginsCore.TaskExecutionContext, k8sPluginState *k8s.PluginState, k8sReader client.Reader) *pluginContext {
+ return &pluginContext{
+ TaskExecutionContext: tCtx,
+ k8sPluginState: k8sPluginState,
+ k8sReader: k8sReader,
+ }
+}
+
+// pluginStateReader is a specialized PluginStateReader that returns a pre-assigned k8s.PluginState.
+// This allows the PluginManager to encapsulate state persistence and only expose Phase/PhaseVersion/Reason
+// to k8s plugins.
+type pluginStateReader struct {
+ k8sPluginState *k8s.PluginState
+}
+
+func (p pluginStateReader) GetStateVersion() uint8 {
+ return 0
+}
+
+func (p pluginStateReader) Get(t interface{}) (uint8, error) {
+ if pointer, ok := t.(*k8s.PluginState); ok {
+ *pointer = *p.k8sPluginState
+ } else {
+ return 0, fmt.Errorf("unexpected type when reading plugin state: expected *k8s.PluginState, got %T", t)
+ }
+ return 0, nil
+}
diff --git a/executor/pkg/plugin/k8s/plugin_manager.go b/executor/pkg/plugin/k8s/plugin_manager.go
new file mode 100644
index 00000000000..d9325d0ef6e
--- /dev/null
+++ b/executor/pkg/plugin/k8s/plugin_manager.go
@@ -0,0 +1,432 @@
+package k8s
+
+import (
+ "context"
+ "fmt"
+ "sync"
+ "time"
+
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/io"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/ioutils"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+ k8serrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ k8stypes "k8s.io/apimachinery/pkg/types"
+ "k8s.io/apimachinery/pkg/util/validation"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/errors"
+ pluginsCore "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/core"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/flytek8s/config"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/k8s"
+ pluginsUtils "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/utils"
+ stdErrors "github.com/flyteorg/flyte/v2/flytestdlib/errors"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+)
+
+const pluginStateVersion = 1
+
+// PluginPhase tracks the high-level phase of the PluginManager's state machine.
+type PluginPhase uint8
+
+const (
+ PluginPhaseNotStarted PluginPhase = iota
+ PluginPhaseStarted
+)
+
+// PluginState is the state persisted by the PluginManager between reconciliation rounds.
+type PluginState struct {
+ Phase PluginPhase
+ K8sPluginState k8s.PluginState
+ LastEventUpdate time.Time
+ LastEventRecordedAt time.Time
+}
+
+var _ pluginsCore.Plugin = &PluginManager{}
+
+// PluginManager wraps a k8s.Plugin to implement pluginsCore.Plugin. It manages the lifecycle
+// of creating, monitoring, aborting, and finalizing Kubernetes resources for task execution.
+type PluginManager struct {
+ id string
+ plugin k8s.Plugin
+ kubeClient pluginsCore.KubeClient
+
+ eventWatcher objectEventWatcher
+ eventWatcherOnce sync.Once
+ eventWatcherErr error
+}
+
+// NewPluginManager creates a PluginManager that wraps a k8s.Plugin.
+func NewPluginManager(id string, plugin k8s.Plugin, kubeClient pluginsCore.KubeClient) *PluginManager {
+ return &PluginManager{
+ id: id,
+ plugin: plugin,
+ kubeClient: kubeClient,
+ }
+}
+
+func (pm *PluginManager) GetID() string {
+ return pm.id
+}
+
+func (pm *PluginManager) GetProperties() pluginsCore.PluginProperties {
+ props := pm.plugin.GetProperties()
+ return pluginsCore.PluginProperties{
+ GeneratedNameMaxLength: props.GeneratedNameMaxLength,
+ }
+}
+
+func (pm *PluginManager) addObjectMetadata(taskCtx pluginsCore.TaskExecutionMetadata, o client.Object, cfg *config.K8sPluginConfig) {
+ o.SetNamespace(taskCtx.GetNamespace())
+ o.SetAnnotations(pluginsUtils.UnionMaps(cfg.DefaultAnnotations, o.GetAnnotations(), pluginsUtils.CopyMap(taskCtx.GetAnnotations())))
+ o.SetLabels(pluginsUtils.UnionMaps(cfg.DefaultLabels, o.GetLabels(), pluginsUtils.CopyMap(taskCtx.GetLabels())))
+ o.SetName(taskCtx.GetTaskExecutionID().GetGeneratedName())
+
+ if !pm.plugin.GetProperties().DisableInjectOwnerReferences && !cfg.DisableInjectOwnerReferences {
+ o.SetOwnerReferences([]metav1.OwnerReference{taskCtx.GetOwnerReference()})
+ }
+
+ if cfg.InjectFinalizer && !pm.plugin.GetProperties().DisableInjectFinalizer {
+ f := append(o.GetFinalizers(), "flyte/flytek8s")
+ o.SetFinalizers(f)
+ }
+
+ if errs := validation.IsDNS1123Subdomain(o.GetName()); len(errs) > 0 {
+ o.SetName(pluginsUtils.ConvertToDNS1123SubdomainCompatibleString(o.GetName()))
+ }
+}
+
+func (pm *PluginManager) launchResource(ctx context.Context, tCtx pluginsCore.TaskExecutionContext) (pluginsCore.Transition, error) {
+ o, err := pm.plugin.BuildResource(ctx, tCtx)
+ if err != nil {
+ return pluginsCore.UnknownTransition, err
+ }
+
+ pm.addObjectMetadata(tCtx.TaskExecutionMetadata(), o, config.GetK8sPluginConfig())
+ logger.Infof(ctx, "Creating Object: Type:[%v], Object:[%v/%v]", o.GetObjectKind().GroupVersionKind(), o.GetNamespace(), o.GetName())
+
+ err = pm.kubeClient.GetClient().Create(ctx, o)
+ if err != nil && !k8serrors.IsAlreadyExists(err) {
+ if k8serrors.IsForbidden(err) {
+ return pluginsCore.DoTransition(pluginsCore.PhaseInfoRetryableFailure("RuntimeFailure", err.Error(), nil)), nil
+ }
+ if k8serrors.IsRequestEntityTooLargeError(err) {
+ return pluginsCore.DoTransition(pluginsCore.PhaseInfoFailure("EntityTooLarge", err.Error(), nil)), nil
+ }
+ reason := k8serrors.ReasonForError(err)
+ logger.Errorf(ctx, "Failed to launch job, system error. err: %v", err)
+ return pluginsCore.UnknownTransition, errors.Wrapf(stdErrors.ErrorCode(reason), err, "failed to create resource")
+ }
+
+ return pluginsCore.DoTransition(pluginsCore.PhaseInfoQueued(time.Now(), pluginsCore.DefaultPhaseVersion, "task submitted to K8s")), nil
+}
+
+func (pm *PluginManager) getResource(ctx context.Context, tCtx pluginsCore.TaskExecutionContext) (client.Object, error) {
+ o, err := pm.plugin.BuildIdentityResource(ctx, tCtx.TaskExecutionMetadata())
+ if err != nil {
+ logger.Errorf(ctx, "Failed to build the Resource with name: %v. Error: %v",
+ tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetGeneratedName(), err)
+ return nil, err
+ }
+ pm.addObjectMetadata(tCtx.TaskExecutionMetadata(), o, config.GetK8sPluginConfig())
+ return o, nil
+}
+
+func (pm *PluginManager) checkResourcePhase(ctx context.Context, tCtx pluginsCore.TaskExecutionContext, o client.Object, k8sPluginState *k8s.PluginState) (pluginsCore.Transition, error) {
+ nsName := k8stypes.NamespacedName{Namespace: o.GetNamespace(), Name: o.GetName()}
+
+ if err := pm.kubeClient.GetClient().Get(ctx, nsName, o); err != nil {
+ if k8serrors.IsNotFound(err) || k8serrors.IsGone(err) || k8serrors.IsResourceExpired(err) {
+ logger.Warningf(ctx, "Failed to find the Resource with name: %v. Error: %v", nsName, err)
+ failureReason := fmt.Sprintf("resource not found, name [%s]. reason: %s", nsName.String(), err.Error())
+ return pluginsCore.DoTransition(pluginsCore.PhaseInfoSystemRetryableFailure("ResourceDeletedExternally", failureReason, nil)), nil
+ }
+ logger.Warningf(ctx, "Failed to retrieve Resource Details with name: %v. Error: %v", nsName, err)
+ return pluginsCore.UnknownTransition, err
+ }
+
+ pCtx := newPluginContext(tCtx, k8sPluginState, pm.kubeClient.GetClient())
+ p, err := pm.plugin.GetTaskPhase(ctx, pCtx, o)
+ if err != nil {
+ logger.Warnf(ctx, "failed to check status of resource in plugin [%s], with error: %s", pm.GetID(), err.Error())
+ return pluginsCore.UnknownTransition, err
+ }
+
+ if p.Phase() == k8sPluginState.Phase && p.Version() < k8sPluginState.PhaseVersion {
+ p = p.WithVersion(k8sPluginState.PhaseVersion)
+ }
+
+ if p.Phase() == pluginsCore.PhaseSuccess {
+ var opReader io.OutputReader
+ if pCtx.ow == nil {
+ opReader = ioutils.NewRemoteFileOutputReader(ctx, tCtx.DataStore(), tCtx.OutputWriter(), 0)
+ } else {
+ opReader = pCtx.ow.GetReader()
+ }
+ y, err := opReader.IsError(ctx)
+ if err != nil {
+ return pluginsCore.UnknownTransition, err
+ }
+ if y {
+ taskErr, err := opReader.ReadError(ctx)
+ if err != nil {
+ return pluginsCore.UnknownTransition, err
+ }
+
+ if taskErr.ExecutionError == nil {
+ taskErr.ExecutionError = &core.ExecutionError{Kind: core.ExecutionError_UNKNOWN, Code: "Unknown", Message: "Unknown"}
+ }
+ var phase pluginsCore.Phase
+ if taskErr.IsRecoverable {
+ phase = pluginsCore.PhaseRetryableFailure
+ } else {
+ phase = pluginsCore.PhasePermanentFailure
+ }
+ return pluginsCore.DoTransitionType(
+ pluginsCore.TransitionTypeEphemeral,
+ pluginsCore.PhaseInfoFailed(phase, taskErr.ExecutionError, p.Info()),
+ ), nil
+ }
+
+ if err := tCtx.OutputWriter().Put(ctx, opReader); err != nil {
+ return pluginsCore.UnknownTransition, err
+ }
+ return pluginsCore.DoTransition(p), nil
+ }
+
+ if !p.Phase().IsTerminal() && o.GetDeletionTimestamp() != nil {
+ failureReason := fmt.Sprintf("object [%s] terminated unexpectedly in the background", nsName.String())
+ return pluginsCore.DoTransition(pluginsCore.PhaseInfoSystemRetryableFailure("UnexpectedObjectDeletion", failureReason, nil)), nil
+ }
+
+ return pluginsCore.DoTransition(p), nil
+}
+
+// Handle implements pluginsCore.Plugin. It is invoked for every reconciliation round.
+func (pm *PluginManager) Handle(ctx context.Context, tCtx pluginsCore.TaskExecutionContext) (pluginsCore.Transition, error) {
+ pluginState := PluginState{}
+ if v, err := tCtx.PluginStateReader().Get(&pluginState); err != nil {
+ if v != pluginStateVersion {
+ return pluginsCore.DoTransition(pluginsCore.PhaseInfoRetryableFailure(errors.CorruptedPluginState,
+ fmt.Sprintf("plugin state version mismatch expected [%d] got [%d]", pluginStateVersion, v), nil)), nil
+ }
+ return pluginsCore.UnknownTransition, errors.Wrapf(errors.CorruptedPluginState, err, "Failed to read unmarshal custom state")
+ }
+
+ var err error
+ var transition pluginsCore.Transition
+ pluginPhase := pluginState.Phase
+ var resource client.Object
+
+ if pluginState.Phase == PluginPhaseNotStarted {
+ transition, err = pm.launchResource(ctx, tCtx)
+ if err == nil && transition.Info().Phase() == pluginsCore.PhaseQueued {
+ pluginPhase = PluginPhaseStarted
+ }
+ } else {
+ o, getErr := pm.getResource(ctx, tCtx)
+ if getErr != nil {
+ transition, err = pluginsCore.DoTransition(pluginsCore.PhaseInfoFailure("BadTaskDefinition",
+ fmt.Sprintf("Failed to build resource, caused by: %s", getErr.Error()), nil)), nil
+ } else {
+ resource = o
+ transition, err = pm.checkResourcePhase(ctx, tCtx, o, &pluginState.K8sPluginState)
+ }
+ }
+
+ if err != nil {
+ return transition, err
+ }
+
+ phaseInfo := transition.Info()
+ lastEventUpdate := pluginState.LastEventUpdate
+ lastEventRecordedAt := pluginState.LastEventRecordedAt
+ if resource != nil {
+ phaseInfo, lastEventUpdate, lastEventRecordedAt = pm.attachRecentObjectEvents(
+ resource,
+ phaseInfo,
+ pluginState.K8sPluginState,
+ lastEventUpdate,
+ lastEventRecordedAt,
+ )
+ transition.SetInfo(phaseInfo)
+ }
+
+ newPluginState := PluginState{
+ Phase: pluginPhase,
+ K8sPluginState: k8s.PluginState{
+ Phase: phaseInfo.Phase(),
+ PhaseVersion: phaseInfo.Version(),
+ Reason: phaseInfo.Reason(),
+ },
+ LastEventUpdate: lastEventUpdate,
+ LastEventRecordedAt: lastEventRecordedAt,
+ }
+ if pluginState != newPluginState {
+ if err := tCtx.PluginStateWriter().Put(pluginStateVersion, &newPluginState); err != nil {
+ return pluginsCore.UnknownTransition, err
+ }
+ }
+
+ return transition, nil
+}
+
+func (pm *PluginManager) initEventWatcher(ctx context.Context) {
+ if pm.eventWatcher != nil {
+ return
+ }
+
+ pm.eventWatcherOnce.Do(func() {
+ pm.eventWatcher, pm.eventWatcherErr = newControllerRuntimeEventWatcher(ctx, pm.kubeClient.GetCache())
+ if pm.eventWatcherErr != nil {
+ logger.Warnf(ctx, "Failed to initialize k8s object event watcher for plugin [%s]: %v", pm.GetID(), pm.eventWatcherErr)
+ }
+ })
+}
+
+// InitializeObjectEventWatcher starts watching Kubernetes object events for this plugin.
+// It is intended to be called during plugin initialization (before task handling starts).
+func (pm *PluginManager) InitializeObjectEventWatcher(ctx context.Context) error {
+ pm.initEventWatcher(ctx)
+ if pm.eventWatcherErr != nil {
+ return fmt.Errorf("failed to initialize k8s object event watcher for plugin %s: %w", pm.GetID(), pm.eventWatcherErr)
+ }
+ return nil
+}
+
+func (pm *PluginManager) attachRecentObjectEvents(
+ resource client.Object,
+ phaseInfo pluginsCore.PhaseInfo,
+ lastObservedState k8s.PluginState,
+ lastEventUpdate time.Time,
+ lastEventRecordedAt time.Time,
+) (pluginsCore.PhaseInfo, time.Time, time.Time) {
+ if pm.eventWatcher == nil || resource == nil {
+ return phaseInfo, lastEventUpdate, lastEventRecordedAt
+ }
+
+ info := phaseInfo.Info()
+ if info == nil {
+ return phaseInfo, lastEventUpdate, lastEventRecordedAt
+ }
+
+ objectKey := watchedObjectKey{
+ Namespace: resource.GetNamespace(),
+ Name: resource.GetName(),
+ Kind: resource.GetObjectKind().GroupVersionKind().Kind,
+ }
+ recentEvents := pm.eventWatcher.List(objectKey, lastEventUpdate, lastEventRecordedAt)
+ if len(recentEvents) == 0 {
+ return phaseInfo, lastEventUpdate, lastEventRecordedAt
+ }
+
+ for _, event := range recentEvents {
+ info.AdditionalReasons = append(info.AdditionalReasons, pluginsCore.ReasonInfo{
+ Reason: event.Message,
+ OccurredAt: &event.CreatedAt,
+ })
+ lastEventUpdate = event.CreatedAt
+ lastEventRecordedAt = event.RecordedAt
+ }
+
+ if phaseInfo.Phase() == lastObservedState.Phase && phaseInfo.Version() <= lastObservedState.PhaseVersion {
+ phaseInfo = phaseInfo.WithVersion(lastObservedState.PhaseVersion + 1)
+ }
+
+ return phaseInfo, lastEventUpdate, lastEventRecordedAt
+}
+
+// Abort implements pluginsCore.Plugin. Called when the task should be killed/aborted.
+func (pm *PluginManager) Abort(ctx context.Context, tCtx pluginsCore.TaskExecutionContext) error {
+ logger.Infof(ctx, "KillTask invoked. We will attempt to delete object [%v].",
+ tCtx.TaskExecutionMetadata().GetTaskExecutionID().GetGeneratedName())
+
+ o, err := pm.getResource(ctx, tCtx)
+ if err != nil {
+ logger.Errorf(ctx, "%v", err)
+ return nil
+ }
+
+ deleteResource := true
+ abortOverride, hasAbortOverride := pm.plugin.(k8s.PluginAbortOverride)
+
+ resourceToFinalize := o
+ var behavior k8s.AbortBehavior
+
+ if hasAbortOverride {
+ behavior, err = abortOverride.OnAbort(ctx, tCtx, o)
+ deleteResource = err == nil && behavior.DeleteResource
+ if err == nil && behavior.Resource != nil {
+ resourceToFinalize = behavior.Resource
+ }
+ }
+
+ if err != nil {
+ // fall through to error check below
+ } else if deleteResource {
+ err = pm.kubeClient.GetClient().Delete(ctx, resourceToFinalize)
+ } else {
+ if behavior.Patch != nil && behavior.Update == nil {
+ err = pm.kubeClient.GetClient().Patch(ctx, resourceToFinalize, behavior.Patch.Patch, behavior.Patch.Options...)
+ } else if behavior.Patch == nil && behavior.Update != nil {
+ err = pm.kubeClient.GetClient().Update(ctx, resourceToFinalize, behavior.Update.Options...)
+ } else {
+ err = fmt.Errorf("AbortBehavior for resource %v must specify either a Patch or an Update operation if Delete is set to false", resourceToFinalize.GetName())
+ }
+ if behavior.DeleteOnErr && err != nil {
+ logger.Warningf(ctx, "Failed to apply AbortBehavior for resource %v with error %v. Will attempt to delete.", resourceToFinalize.GetName(), err)
+ err = pm.kubeClient.GetClient().Delete(ctx, resourceToFinalize)
+ }
+ }
+
+ if err != nil && !k8serrors.IsNotFound(err) && !k8serrors.IsGone(err) {
+ logger.Warningf(ctx, "Failed to abort Resource with name: %v/%v. Error: %v",
+ resourceToFinalize.GetNamespace(), resourceToFinalize.GetName(), err)
+ return err
+ }
+
+ return nil
+}
+
+// Finalize implements pluginsCore.Plugin. Called after Handle or Abort to clean up resources.
+func (pm *PluginManager) Finalize(ctx context.Context, tCtx pluginsCore.TaskExecutionContext) error {
+ o, err := pm.getResource(ctx, tCtx)
+ if err != nil {
+ logger.Errorf(ctx, "%v", err)
+ return nil
+ }
+
+ nsName := k8stypes.NamespacedName{Namespace: o.GetNamespace(), Name: o.GetName()}
+
+ // Clear finalizers
+ if err := pm.kubeClient.GetClient().Get(ctx, nsName, o); err != nil {
+ if k8serrors.IsNotFound(err) || k8serrors.IsGone(err) {
+ return nil
+ }
+ return err
+ }
+
+ if len(o.GetFinalizers()) > 0 {
+ o.SetFinalizers([]string{})
+ if err := pm.kubeClient.GetClient().Update(ctx, o); err != nil {
+ if k8serrors.IsNotFound(err) || k8serrors.IsGone(err) {
+ return nil
+ }
+ logger.Warningf(ctx, "Failed to clear finalizers for Resource: %v. Error: %v", nsName, err)
+ return err
+ }
+ }
+
+ cfg := config.GetK8sPluginConfig()
+ if cfg.DeleteResourceOnFinalize && !pm.plugin.GetProperties().DisableDeleteResourceOnFinalize {
+ if err := pm.kubeClient.GetClient().Delete(ctx, o); err != nil {
+ if k8serrors.IsNotFound(err) || k8serrors.IsGone(err) {
+ return nil
+ }
+ logger.Warningf(ctx, "Failed to delete Resource: %v. Error: %v", nsName, err)
+ return err
+ }
+ }
+
+ return nil
+}
diff --git a/executor/pkg/plugin/registry.go b/executor/pkg/plugin/registry.go
new file mode 100644
index 00000000000..ae924fedbf8
--- /dev/null
+++ b/executor/pkg/plugin/registry.go
@@ -0,0 +1,125 @@
+package plugin
+
+import (
+ "context"
+ "fmt"
+ "sync"
+
+ pluginsCore "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/core"
+ k8sPlugin "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/k8s"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+
+ executorK8s "github.com/flyteorg/flyte/v2/executor/pkg/plugin/k8s"
+)
+
+// PluginRegistryIface provides access to registered plugin entries.
+type PluginRegistryIface interface {
+ GetCorePlugins() []pluginsCore.PluginEntry
+ GetK8sPlugins() []k8sPlugin.PluginEntry
+}
+
+// Registry resolves a pluginsCore.Plugin for a given task type by wrapping the global
+// plugin registry. K8s plugins are wrapped in a PluginManager; core plugins are loaded
+// via their PluginLoader.
+type Registry struct {
+ mu sync.RWMutex
+
+ setupCtx pluginsCore.SetupContext
+ pluginRegistry PluginRegistryIface
+
+ // taskType -> pluginsCore.Plugin
+ plugins map[string]pluginsCore.Plugin
+ defaultPlugin pluginsCore.Plugin
+ initialized bool
+}
+
+// NewRegistry creates a Registry backed by the given plugin source and setup context.
+func NewRegistry(setupCtx pluginsCore.SetupContext, pluginRegistry PluginRegistryIface) *Registry {
+ return &Registry{
+ setupCtx: setupCtx,
+ pluginRegistry: pluginRegistry,
+ plugins: make(map[string]pluginsCore.Plugin),
+ }
+}
+
+// Initialize loads all registered plugins. Must be called once during startup.
+func (r *Registry) Initialize(ctx context.Context) error {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+
+ if r.initialized {
+ return nil
+ }
+
+ // Load k8s plugins
+ for _, entry := range r.pluginRegistry.GetK8sPlugins() {
+ pm := executorK8s.NewPluginManager(
+ entry.ID,
+ entry.Plugin,
+ r.setupCtx.KubeClient(),
+ )
+ if err := pm.InitializeObjectEventWatcher(ctx); err != nil {
+ return fmt.Errorf("failed to initialize k8s object event watcher for plugin %s: %w", entry.ID, err)
+ }
+
+ for _, taskType := range entry.RegisteredTaskTypes {
+ if existing, ok := r.plugins[taskType]; ok {
+ logger.Warnf(ctx, "Task type %q already registered by plugin %q, overwriting with %q",
+ taskType, existing.GetID(), entry.ID)
+ }
+ r.plugins[taskType] = pm
+ }
+
+ if entry.IsDefault {
+ if r.defaultPlugin != nil {
+ logger.Warnf(ctx, "Multiple default plugins found, overwriting %q with %q",
+ r.defaultPlugin.GetID(), entry.ID)
+ }
+ r.defaultPlugin = pm
+ }
+
+ logger.Infof(ctx, "Registered k8s plugin [%s] for task types %v", entry.ID, entry.RegisteredTaskTypes)
+ }
+
+ // Load core plugins
+ for _, entry := range r.pluginRegistry.GetCorePlugins() {
+ plugin, err := pluginsCore.LoadPlugin(ctx, r.setupCtx, entry)
+ if err != nil {
+ return fmt.Errorf("failed to load core plugin %s: %w", entry.ID, err)
+ }
+
+ for _, taskType := range entry.RegisteredTaskTypes {
+ if existing, ok := r.plugins[taskType]; ok {
+ logger.Warnf(ctx, "Task type %q already registered by plugin %q, overwriting with %q",
+ taskType, existing.GetID(), entry.ID)
+ }
+ r.plugins[taskType] = plugin
+ }
+
+ if entry.IsDefault && r.defaultPlugin == nil {
+ r.defaultPlugin = plugin
+ }
+
+ logger.Infof(ctx, "Registered core plugin [%s] for task types %v", entry.ID, entry.RegisteredTaskTypes)
+ }
+
+ r.initialized = true
+ return nil
+}
+
+// ResolvePlugin returns the plugin registered for the given task type.
+// Falls back to the default plugin if no specific match is found.
+func (r *Registry) ResolvePlugin(taskType string) (pluginsCore.Plugin, error) {
+ r.mu.RLock()
+ defer r.mu.RUnlock()
+
+ if p, ok := r.plugins[taskType]; ok {
+ return p, nil
+ }
+
+ if r.defaultPlugin != nil {
+ return r.defaultPlugin, nil
+ }
+
+ return nil, fmt.Errorf("no plugin registered for task type %q and no default plugin available", taskType)
+}
diff --git a/executor/pkg/plugin/setup_context.go b/executor/pkg/plugin/setup_context.go
new file mode 100644
index 00000000000..a2c2f69028b
--- /dev/null
+++ b/executor/pkg/plugin/setup_context.go
@@ -0,0 +1,47 @@
+package plugin
+
+import (
+ pluginsCore "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/core"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+)
+
+var _ pluginsCore.SetupContext = &setupContext{}
+
+type setupContext struct {
+ kubeClient pluginsCore.KubeClient
+ secretManager pluginsCore.SecretManager
+ resourceRegistrar pluginsCore.ResourceRegistrar
+ enqueueOwner pluginsCore.EnqueueOwner
+ enqueueLabels []string
+ ownerKind string
+ metricsScope promutils.Scope
+}
+
+// NewSetupContext creates a SetupContext implementation for the executor.
+func NewSetupContext(
+ kubeClient pluginsCore.KubeClient,
+ secretManager pluginsCore.SecretManager,
+ resourceRegistrar pluginsCore.ResourceRegistrar,
+ enqueueOwner pluginsCore.EnqueueOwner,
+ enqueueLabels []string,
+ ownerKind string,
+ metricsScope promutils.Scope,
+) pluginsCore.SetupContext {
+ return &setupContext{
+ kubeClient: kubeClient,
+ secretManager: secretManager,
+ resourceRegistrar: resourceRegistrar,
+ enqueueOwner: enqueueOwner,
+ enqueueLabels: enqueueLabels,
+ ownerKind: ownerKind,
+ metricsScope: metricsScope,
+ }
+}
+
+func (s *setupContext) EnqueueOwner() pluginsCore.EnqueueOwner { return s.enqueueOwner }
+func (s *setupContext) IncludeEnqueueLabels() []string { return s.enqueueLabels }
+func (s *setupContext) OwnerKind() string { return s.ownerKind }
+func (s *setupContext) MetricsScope() promutils.Scope { return s.metricsScope }
+func (s *setupContext) KubeClient() pluginsCore.KubeClient { return s.kubeClient }
+func (s *setupContext) SecretManager() pluginsCore.SecretManager { return s.secretManager }
+func (s *setupContext) ResourceRegistrar() pluginsCore.ResourceRegistrar { return s.resourceRegistrar }
diff --git a/executor/pkg/plugin/state_manager.go b/executor/pkg/plugin/state_manager.go
new file mode 100644
index 00000000000..f6ab2ba3ea8
--- /dev/null
+++ b/executor/pkg/plugin/state_manager.go
@@ -0,0 +1,81 @@
+package plugin
+
+import (
+ "bytes"
+ "encoding/gob"
+ "fmt"
+
+ pluginsCore "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/core"
+)
+
+var (
+ _ pluginsCore.PluginStateReader = &PluginStateManager{}
+ _ pluginsCore.PluginStateWriter = &PluginStateManager{}
+)
+
+// PluginStateManager implements PluginStateReader and PluginStateWriter using Gob encoding
+// over byte buffers. It is initialized with the previous state from TaskAction.Status.PluginState
+// and captures new state written by the plugin during Handle.
+type PluginStateManager struct {
+ prevStateBytes []byte
+ prevStateVersion uint8
+
+ newStateBytes []byte
+ newStateVersion uint8
+ stateWritten bool
+}
+
+// NewPluginStateManager creates a PluginStateManager initialized with the previous round's state.
+func NewPluginStateManager(prevState []byte, prevVersion uint8) *PluginStateManager {
+ return &PluginStateManager{
+ prevStateBytes: prevState,
+ prevStateVersion: prevVersion,
+ }
+}
+
+// GetStateVersion returns the version of the previous state.
+func (m *PluginStateManager) GetStateVersion() uint8 {
+ return m.prevStateVersion
+}
+
+// Get deserializes the previous state into t using Gob decoding.
+// If there is no previous state, t remains its zero value and version 0 is returned.
+func (m *PluginStateManager) Get(t interface{}) (uint8, error) {
+ if len(m.prevStateBytes) == 0 {
+ return 0, nil
+ }
+ buf := bytes.NewBuffer(m.prevStateBytes)
+ dec := gob.NewDecoder(buf)
+ if err := dec.Decode(t); err != nil {
+ return 0, fmt.Errorf("failed to decode plugin state: %w", err)
+ }
+ return m.prevStateVersion, nil
+}
+
+// Put serializes v using Gob encoding and stores it as the new state.
+// Only the last call to Put is recorded; all previous calls are overwritten.
+func (m *PluginStateManager) Put(stateVersion uint8, v interface{}) error {
+ var buf bytes.Buffer
+ enc := gob.NewEncoder(&buf)
+ if err := enc.Encode(v); err != nil {
+ return fmt.Errorf("failed to encode plugin state: %w", err)
+ }
+ m.newStateBytes = buf.Bytes()
+ m.newStateVersion = stateVersion
+ m.stateWritten = true
+ return nil
+}
+
+// Reset clears the state to empty.
+func (m *PluginStateManager) Reset() error {
+ m.newStateBytes = nil
+ m.newStateVersion = 0
+ m.stateWritten = true
+ return nil
+}
+
+// GetNewState returns the new state bytes, version, and whether state was written during this round.
+// The controller uses this to persist the state back to TaskAction.Status.
+func (m *PluginStateManager) GetNewState() (stateBytes []byte, version uint8, written bool) {
+ return m.newStateBytes, m.newStateVersion, m.stateWritten
+}
diff --git a/executor/pkg/plugin/state_manager_test.go b/executor/pkg/plugin/state_manager_test.go
new file mode 100644
index 00000000000..300b56922b2
--- /dev/null
+++ b/executor/pkg/plugin/state_manager_test.go
@@ -0,0 +1,90 @@
+package plugin
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ pluginsCore "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/core"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/k8s"
+)
+
+func TestPluginStateManager_EmptyState(t *testing.T) {
+ mgr := NewPluginStateManager(nil, 0)
+
+ assert.Equal(t, uint8(0), mgr.GetStateVersion())
+
+ var state k8s.PluginState
+ version, err := mgr.Get(&state)
+ require.NoError(t, err)
+ assert.Equal(t, uint8(0), version)
+ assert.Equal(t, k8s.PluginState{}, state)
+}
+
+func TestPluginStateManager_RoundTrip(t *testing.T) {
+ original := k8s.PluginState{
+ Phase: pluginsCore.PhaseRunning,
+ PhaseVersion: 3,
+ Reason: "task is running",
+ }
+
+ // Write state
+ writeMgr := NewPluginStateManager(nil, 0)
+ require.NoError(t, writeMgr.Put(1, &original))
+
+ newBytes, newVersion, written := writeMgr.GetNewState()
+ assert.True(t, written)
+ assert.Equal(t, uint8(1), newVersion)
+ assert.NotEmpty(t, newBytes)
+
+ // Read state back in a new manager (simulates next reconciliation round)
+ readMgr := NewPluginStateManager(newBytes, newVersion)
+ assert.Equal(t, uint8(1), readMgr.GetStateVersion())
+
+ var restored k8s.PluginState
+ version, err := readMgr.Get(&restored)
+ require.NoError(t, err)
+ assert.Equal(t, uint8(1), version)
+ assert.Equal(t, original, restored)
+}
+
+func TestPluginStateManager_PutOverwrites(t *testing.T) {
+ mgr := NewPluginStateManager(nil, 0)
+
+ first := k8s.PluginState{Phase: pluginsCore.PhaseQueued}
+ second := k8s.PluginState{Phase: pluginsCore.PhaseRunning}
+
+ require.NoError(t, mgr.Put(1, &first))
+ require.NoError(t, mgr.Put(2, &second))
+
+ newBytes, newVersion, written := mgr.GetNewState()
+ assert.True(t, written)
+ assert.Equal(t, uint8(2), newVersion)
+
+ readMgr := NewPluginStateManager(newBytes, newVersion)
+ var restored k8s.PluginState
+ _, err := readMgr.Get(&restored)
+ require.NoError(t, err)
+ assert.Equal(t, pluginsCore.PhaseRunning, restored.Phase)
+}
+
+func TestPluginStateManager_Reset(t *testing.T) {
+ original := k8s.PluginState{Phase: pluginsCore.PhaseRunning}
+
+ writeMgr := NewPluginStateManager(nil, 0)
+ require.NoError(t, writeMgr.Put(1, &original))
+ require.NoError(t, writeMgr.Reset())
+
+ newBytes, newVersion, written := writeMgr.GetNewState()
+ assert.True(t, written)
+ assert.Equal(t, uint8(0), newVersion)
+ assert.Nil(t, newBytes)
+}
+
+func TestPluginStateManager_NoWriteReturnsNotWritten(t *testing.T) {
+ mgr := NewPluginStateManager(nil, 0)
+
+ _, _, written := mgr.GetNewState()
+ assert.False(t, written)
+}
diff --git a/executor/pkg/plugin/task_exec_context.go b/executor/pkg/plugin/task_exec_context.go
new file mode 100644
index 00000000000..62325d18471
--- /dev/null
+++ b/executor/pkg/plugin/task_exec_context.go
@@ -0,0 +1,170 @@
+package plugin
+
+import (
+ "context"
+ "fmt"
+ "net/url"
+ "strconv"
+ "strings"
+
+ "google.golang.org/protobuf/proto"
+
+ flyteorgv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/catalog"
+ pluginsCore "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/core"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/io"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/ioutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+var _ pluginsCore.TaskExecutionContext = &taskExecutionContext{}
+
+type taskExecutionContext struct {
+ resourceManager pluginsCore.ResourceManager
+ secretManager pluginsCore.SecretManager
+ taskRefreshIndicator pluginsCore.SignalAsync
+ dataStore *storage.DataStore
+ pluginStateReader pluginsCore.PluginStateReader
+ pluginStateWriter pluginsCore.PluginStateWriter
+ taskReader pluginsCore.TaskReader
+ inputReader io.InputReader
+ taskExecMetadata pluginsCore.TaskExecutionMetadata
+ outputWriter io.OutputWriter
+ catalogClient catalog.AsyncClient
+}
+
+func (t *taskExecutionContext) ResourceManager() pluginsCore.ResourceManager {
+ return t.resourceManager
+}
+func (t *taskExecutionContext) SecretManager() pluginsCore.SecretManager { return t.secretManager }
+func (t *taskExecutionContext) TaskRefreshIndicator() pluginsCore.SignalAsync {
+ return t.taskRefreshIndicator
+}
+func (t *taskExecutionContext) DataStore() *storage.DataStore { return t.dataStore }
+func (t *taskExecutionContext) PluginStateReader() pluginsCore.PluginStateReader {
+ return t.pluginStateReader
+}
+func (t *taskExecutionContext) PluginStateWriter() pluginsCore.PluginStateWriter {
+ return t.pluginStateWriter
+}
+func (t *taskExecutionContext) TaskReader() pluginsCore.TaskReader { return t.taskReader }
+func (t *taskExecutionContext) InputReader() io.InputReader { return t.inputReader }
+func (t *taskExecutionContext) TaskExecutionMetadata() pluginsCore.TaskExecutionMetadata {
+ return t.taskExecMetadata
+}
+func (t *taskExecutionContext) OutputWriter() io.OutputWriter { return t.outputWriter }
+func (t *taskExecutionContext) Catalog() catalog.AsyncClient { return t.catalogClient }
+
+// ComputeActionOutputPath constructs the full output directory for a task action:
+//
+// ://////
+//
+// The shard is a 2-char base-36 prefix derived deterministically from the
+// TaskAction's namespace and name. It is inserted immediately after the bucket
+// (AWS S3 hot-spot avoidance convention) so that storage traffic is spread
+// across key prefixes from the root. The attempt segment isolates each retry.
+func ComputeActionOutputPath(ctx context.Context, namespace, name, runOutputBase, actionName string, attempt uint32) (storage.DataReference, error) {
+ sharder, err := ioutils.NewBase36PrefixShardSelector(ctx)
+ if err != nil {
+ return "", err
+ }
+ shard, err := sharder.GetShardPrefix(ctx, []byte(namespace+"/"+name))
+ if err != nil {
+ return "", err
+ }
+
+ u, err := url.Parse(runOutputBase)
+ if err != nil {
+ return "", fmt.Errorf("invalid RunOutputBase %q: %w", runOutputBase, err)
+ }
+
+ // Insert shard between bucket and the rest of the path:
+ // s3://bucket/org/proj/domain/run/ → s3://bucket//org/proj/domain/run//
+ restPath := strings.Trim(u.Path, "/")
+ segments := []string{shard}
+ if restPath != "" {
+ segments = append(segments, restPath)
+ }
+ segments = append(segments, actionName, strconv.Itoa(int(attempt)))
+ u.Path = "/" + strings.Join(segments, "/")
+
+ return storage.DataReference(u.String()), nil
+}
+
+// inlineTaskReader reads a TaskTemplate from bytes stored inline in the CRD.
+type inlineTaskReader struct {
+ data []byte
+}
+
+func (r *inlineTaskReader) Path(_ context.Context) (storage.DataReference, error) {
+ return "inline://taskTemplate", nil
+}
+
+func (r *inlineTaskReader) Read(_ context.Context) (*core.TaskTemplate, error) {
+ t := &core.TaskTemplate{}
+ if err := proto.Unmarshal(r.data, t); err != nil {
+ return nil, err
+ }
+ return t, nil
+}
+
+// NewTaskExecutionContext creates a TaskExecutionContext from a TaskAction and supporting dependencies.
+func NewTaskExecutionContext(
+ taskAction *flyteorgv1.TaskAction,
+ dataStore *storage.DataStore,
+ pluginStateMgr *PluginStateManager,
+ secretManager pluginsCore.SecretManager,
+ resourceManager pluginsCore.ResourceManager,
+ catalogClient catalog.AsyncClient,
+) (*taskExecutionContext, error) {
+ ctx := context.Background()
+
+ // Task reader (inline from CRD)
+ taskReader := &inlineTaskReader{
+ data: taskAction.Spec.TaskTemplate,
+ }
+
+ // Input reader — InputURI may be a full path (ending in inputs.pb) or just
+ // the prefix directory. NewInputFilePaths always appends "inputs.pb", so
+ // strip a trailing suffix to avoid the doubled "inputs.pb/inputs.pb" path.
+ inputPathPrefix := storage.DataReference(
+ strings.TrimSuffix(taskAction.Spec.InputURI, "/"+ioutils.InputsSuffix),
+ )
+ inputPaths := ioutils.NewInputFilePaths(ctx, dataStore, inputPathPrefix)
+ inputReader := ioutils.NewRemoteFileInputReader(ctx, dataStore, inputPaths)
+
+ // Output writer — scope outputs per action and attempt so retries don't overwrite each other.
+ // Path: ////
+ attempt := taskAction.Status.Attempts
+ if attempt == 0 { // if attempts is not set, default to 1
+ attempt = 1
+ }
+ outputPrefix, err := ComputeActionOutputPath(ctx, taskAction.Namespace, taskAction.Name, taskAction.Spec.RunOutputBase, taskAction.Spec.ActionName, attempt)
+ if err != nil {
+ return nil, err
+ }
+ rawOutputPaths := ioutils.NewRawOutputPaths(ctx, outputPrefix)
+ outputFilePaths := ioutils.NewCheckpointRemoteFilePaths(ctx, dataStore, outputPrefix, rawOutputPaths, "")
+ outputWriter := ioutils.NewRemoteFileOutputWriter(ctx, dataStore, outputFilePaths)
+
+ // Task execution metadata
+ taskExecMeta, err := NewTaskExecutionMetadata(taskAction)
+ if err != nil {
+ return nil, err
+ }
+
+ return &taskExecutionContext{
+ resourceManager: resourceManager,
+ secretManager: secretManager,
+ taskRefreshIndicator: func(_ context.Context) {},
+ dataStore: dataStore,
+ pluginStateReader: pluginStateMgr,
+ pluginStateWriter: pluginStateMgr,
+ taskReader: taskReader,
+ inputReader: inputReader,
+ taskExecMetadata: taskExecMeta,
+ outputWriter: outputWriter,
+ catalogClient: catalogClient,
+ }, nil
+}
diff --git a/executor/pkg/plugin/task_exec_metadata.go b/executor/pkg/plugin/task_exec_metadata.go
new file mode 100644
index 00000000000..271a3cd1852
--- /dev/null
+++ b/executor/pkg/plugin/task_exec_metadata.go
@@ -0,0 +1,240 @@
+package plugin
+
+import (
+ "fmt"
+
+ "google.golang.org/protobuf/proto"
+ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/utils/ptr"
+
+ flyteorgv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+ pluginsCore "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/core"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/flytek8s"
+ pluginsUtils "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/utils"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/utils/secrets"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+var _ pluginsCore.TaskExecutionMetadata = &taskExecutionMetadata{}
+var _ pluginsCore.TaskExecutionID = &taskExecutionID{}
+
+type taskExecutionID struct {
+ generatedName string
+ id core.TaskExecutionIdentifier
+}
+
+func (t *taskExecutionID) GetGeneratedName() string {
+ return t.generatedName
+}
+
+func (t *taskExecutionID) GetGeneratedNameWith(minLength, maxLength int) (string, error) {
+ name := t.generatedName
+ if len(name) > maxLength {
+ name = name[:maxLength]
+ }
+ return name, nil
+}
+
+func (t *taskExecutionID) GetID() *core.TaskExecutionIdentifier {
+ return &t.id
+}
+
+func (t *taskExecutionID) GetUniqueNodeID() string {
+ return t.generatedName
+}
+
+type taskExecutionMetadata struct {
+ ownerID types.NamespacedName
+ taskExecutionID pluginsCore.TaskExecutionID
+ namespace string
+ ownerReference metav1.OwnerReference
+ labels map[string]string
+ annotations map[string]string
+ maxAttempts uint32
+ overrides pluginsCore.TaskOverrides
+ envVars map[string]string
+ interruptible bool
+ securityContext *core.SecurityContext
+}
+
+// NewTaskExecutionMetadata creates a TaskExecutionMetadata from a TaskAction.
+func NewTaskExecutionMetadata(ta *flyteorgv1.TaskAction) (pluginsCore.TaskExecutionMetadata, error) {
+ // Extract resource requirements from the inline task template
+ overrides := buildOverridesFromTaskTemplate(ta.Spec.TaskTemplate)
+
+ // Handling secrets
+ var err error
+ securityContext := extractSecurityContextFromTaskTemplate(ta.Spec.TaskTemplate)
+ secretsMap := make(map[string]string)
+ injectLabels := make(map[string]string)
+ if securityContext != nil && len(securityContext.Secrets) > 0 {
+ secretsMap, err = secrets.MarshalSecretsToMapStrings(securityContext.Secrets)
+ if err != nil {
+ return nil, err
+ }
+ injectLabels[secrets.PodLabel] = secrets.PodLabelValue
+ }
+
+ // Build environment variables for the task pod
+ envVars := map[string]string{
+ "ACTION_NAME": ta.Spec.ActionName,
+ "RUN_NAME": ta.Spec.RunName,
+ "_U_RUN_BASE": ta.Spec.RunOutputBase,
+ "_U_ORG_NAME": "local",
+ }
+ for key, value := range ta.Spec.EnvVars {
+ if _, exists := envVars[key]; !exists {
+ envVars[key] = value
+ }
+ }
+ generatedName := buildGeneratedName(ta)
+ retryAttempt := attemptToRetry(ta.Status.Attempts)
+ maxAttempts := maxAttemptsFromTaskTemplate(ta.Spec.TaskTemplate)
+
+ return &taskExecutionMetadata{
+ ownerID: types.NamespacedName{
+ Name: ta.Name,
+ Namespace: ta.Namespace,
+ },
+ taskExecutionID: &taskExecutionID{
+ generatedName: generatedName,
+ id: core.TaskExecutionIdentifier{
+ NodeExecutionId: &core.NodeExecutionIdentifier{
+ ExecutionId: &core.WorkflowExecutionIdentifier{
+ Project: ta.Spec.Project,
+ Domain: ta.Spec.Domain,
+ Name: ta.Spec.RunName,
+ },
+ NodeId: ta.Spec.ActionName,
+ },
+ RetryAttempt: retryAttempt,
+ },
+ },
+ namespace: ta.Namespace,
+ ownerReference: metav1.OwnerReference{
+ APIVersion: flyteorgv1.GroupVersion.String(),
+ Kind: "TaskAction",
+ Name: ta.Name,
+ UID: ta.UID,
+ Controller: ptr.To(true),
+ },
+ labels: pluginsUtils.UnionMaps(ta.Labels, injectLabels),
+ annotations: pluginsUtils.UnionMaps(ta.Annotations, secretsMap),
+ maxAttempts: maxAttempts,
+ overrides: overrides,
+ envVars: envVars,
+ interruptible: ta.Spec.Interruptible != nil && *ta.Spec.Interruptible,
+ securityContext: securityContext,
+ }, nil
+}
+
+func buildGeneratedName(ta *flyteorgv1.TaskAction) string {
+ return fmt.Sprintf("%s-%d", ta.Name, attemptToRetry(ta.Status.Attempts))
+}
+
+// attemptToRetry convert attempt to retry count
+func attemptToRetry(attempt uint32) uint32 {
+ if attempt <= 1 {
+ return 0
+ }
+ return attempt - 1
+}
+
+// maxAttemptsFromTaskTemplate give the max attempts (retries + 1) from the task template.
+func maxAttemptsFromTaskTemplate(data []byte) uint32 {
+ if len(data) == 0 {
+ return 1
+ }
+
+ tmpl := &core.TaskTemplate{}
+ if err := proto.Unmarshal(data, tmpl); err != nil {
+ return 1
+ }
+
+ md := tmpl.GetMetadata()
+ if md == nil || md.GetRetries() == nil {
+ return 1
+ }
+
+ return md.GetRetries().GetRetries() + 1
+}
+
+// buildOverridesFromTaskTemplate deserializes the task template and extracts resource requirements.
+func buildOverridesFromTaskTemplate(data []byte) *taskOverrides {
+ if len(data) == 0 {
+ return &taskOverrides{}
+ }
+ tmpl := &core.TaskTemplate{}
+ if err := proto.Unmarshal(data, tmpl); err != nil {
+ return &taskOverrides{}
+ }
+ container := tmpl.GetContainer()
+ if container == nil {
+ return &taskOverrides{}
+ }
+ res, err := flytek8s.ToK8sResourceRequirements(container.Resources)
+ if err != nil {
+ return &taskOverrides{}
+ }
+ return &taskOverrides{resources: res}
+}
+
+// extractSecurityContextFromTaskTemplate deserializes the task template and extracts SecurityContext.
+func extractSecurityContextFromTaskTemplate(data []byte) *core.SecurityContext {
+ if len(data) == 0 {
+ return &core.SecurityContext{}
+ }
+ tmpl := &core.TaskTemplate{}
+ if err := proto.Unmarshal(data, tmpl); err != nil {
+ return &core.SecurityContext{}
+ }
+ if tmpl.SecurityContext == nil {
+ return &core.SecurityContext{}
+ }
+ return tmpl.GetSecurityContext()
+}
+
+func (m *taskExecutionMetadata) GetOwnerID() types.NamespacedName { return m.ownerID }
+func (m *taskExecutionMetadata) GetTaskExecutionID() pluginsCore.TaskExecutionID {
+ return m.taskExecutionID
+}
+func (m *taskExecutionMetadata) GetNamespace() string { return m.namespace }
+func (m *taskExecutionMetadata) GetOwnerReference() metav1.OwnerReference { return m.ownerReference }
+func (m *taskExecutionMetadata) GetLabels() map[string]string { return m.labels }
+func (m *taskExecutionMetadata) GetAnnotations() map[string]string { return m.annotations }
+func (m *taskExecutionMetadata) GetMaxAttempts() uint32 { return m.maxAttempts }
+func (m *taskExecutionMetadata) GetK8sServiceAccount() string { return "" }
+func (m *taskExecutionMetadata) IsInterruptible() bool { return m.interruptible }
+func (m *taskExecutionMetadata) GetInterruptibleFailureThreshold() int32 { return 0 }
+func (m *taskExecutionMetadata) GetEnvironmentVariables() map[string]string { return m.envVars }
+func (m *taskExecutionMetadata) GetConsoleURL() string { return "" }
+
+func (m *taskExecutionMetadata) GetOverrides() pluginsCore.TaskOverrides {
+ return m.overrides
+}
+
+func (m *taskExecutionMetadata) GetSecurityContext() *core.SecurityContext {
+ return m.securityContext
+}
+
+func (m *taskExecutionMetadata) GetPlatformResources() *v1.ResourceRequirements {
+ return &v1.ResourceRequirements{}
+}
+
+func (m *taskExecutionMetadata) GetExternalResourceAttributes() pluginsCore.ExternalResourceAttributes {
+ return pluginsCore.ExternalResourceAttributes{}
+}
+
+// taskOverrides provides resource overrides extracted from the task template.
+type taskOverrides struct {
+ resources *v1.ResourceRequirements
+}
+
+func (t *taskOverrides) GetResources() *v1.ResourceRequirements { return t.resources }
+func (t *taskOverrides) GetExtendedResources() *core.ExtendedResources { return nil }
+func (t *taskOverrides) GetContainerImage() string { return "" }
+func (t *taskOverrides) GetConfigMap() *v1.ConfigMap { return nil }
+func (t *taskOverrides) GetPodTemplate() *core.K8SPod { return nil }
+func (t *taskOverrides) GetConfig() map[string]string { return nil }
diff --git a/executor/pkg/plugin/task_exec_metadata_test.go b/executor/pkg/plugin/task_exec_metadata_test.go
new file mode 100644
index 00000000000..57b1da4ec5a
--- /dev/null
+++ b/executor/pkg/plugin/task_exec_metadata_test.go
@@ -0,0 +1,56 @@
+package plugin
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ flyteorgv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+)
+
+func TestNewTaskExecutionMetadata_UsesProjectedRunContext(t *testing.T) {
+ interruptible := true
+ taskAction := &flyteorgv1.TaskAction{
+ Spec: flyteorgv1.TaskActionSpec{
+ Project: "project",
+ Domain: "development",
+ RunName: "run-name",
+ ActionName: "action-name",
+ RunOutputBase: "s3://bucket/run",
+ EnvVars: map[string]string{"TRACE_ID": "root-abc"},
+ Interruptible: &interruptible,
+ },
+ }
+
+ meta, err := NewTaskExecutionMetadata(taskAction)
+ require.NoError(t, err)
+ require.Equal(t, "root-abc", meta.GetEnvironmentVariables()["TRACE_ID"])
+ require.True(t, meta.IsInterruptible())
+}
+
+func TestNewTaskExecutionMetadata_UserEnvVarsCannotClobberInternal(t *testing.T) {
+ taskAction := &flyteorgv1.TaskAction{
+ Spec: flyteorgv1.TaskActionSpec{
+ Project: "project",
+ Domain: "development",
+ RunName: "run-name",
+ ActionName: "action-name",
+ RunOutputBase: "s3://bucket/run",
+ EnvVars: map[string]string{
+ "ACTION_NAME": "malicious-override",
+ "RUN_NAME": "malicious-override",
+ "_U_RUN_BASE": "malicious-override",
+ "USER_VAR": "allowed",
+ },
+ },
+ }
+
+ meta, err := NewTaskExecutionMetadata(taskAction)
+ require.NoError(t, err)
+
+ env := meta.GetEnvironmentVariables()
+ require.Equal(t, "action-name", env["ACTION_NAME"])
+ require.Equal(t, "run-name", env["RUN_NAME"])
+ require.Equal(t, "s3://bucket/run", env["_U_RUN_BASE"])
+ require.Equal(t, "allowed", env["USER_VAR"])
+}
diff --git a/executor/pkg/webhook/entrypoint.go b/executor/pkg/webhook/entrypoint.go
new file mode 100644
index 00000000000..6d54d4fef17
--- /dev/null
+++ b/executor/pkg/webhook/entrypoint.go
@@ -0,0 +1,98 @@
+package webhook
+
+import (
+ "context"
+ errors2 "errors"
+ "fmt"
+ "os"
+
+ "k8s.io/apimachinery/pkg/api/errors"
+ v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes"
+ "sigs.k8s.io/controller-runtime/pkg/manager"
+
+ webhookConfig "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/secret/config"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+)
+
+const (
+ PodNameEnvVar = "POD_NAME"
+ PodNamespaceEnvVar = "POD_NAMESPACE"
+)
+
+// Setup initializes the webhook: generates certs, registers MutatingWebhookConfiguration, and registers the HTTP handler.
+// It is called before mgr.Start() so that the webhook server is ready to receive requests.
+func Setup(ctx context.Context, kubeClient kubernetes.Interface, cfg *webhookConfig.Config,
+ defaultNamespace string, scope promutils.Scope, mgr manager.Manager) error {
+
+ if err := InitCerts(ctx, kubeClient, cfg); err != nil {
+ return fmt.Errorf("webhook: failed to initialize certs: %w", err)
+ }
+
+ podMutator, err := NewPodMutator(ctx, cfg, defaultNamespace, mgr.GetScheme(), scope)
+ if err != nil {
+ return fmt.Errorf("webhook: failed to create pod mutator: %w", err)
+ }
+
+ if err := createMutationConfig(ctx, kubeClient, podMutator, defaultNamespace); err != nil {
+ return fmt.Errorf("webhook: failed to create MutatingWebhookConfiguration: %w", err)
+ }
+
+ if err := podMutator.Register(ctx, mgr); err != nil {
+ return fmt.Errorf("webhook: failed to register handler: %w", err)
+ }
+
+ logger.Infof(ctx, "Webhook setup complete")
+ return nil
+}
+
+func createMutationConfig(ctx context.Context, kubeClient kubernetes.Interface, webhookObj *PodMutator, defaultNamespace string) error {
+ shouldAddOwnerRef := true
+ podName, found := os.LookupEnv(PodNameEnvVar)
+ if !found {
+ shouldAddOwnerRef = false
+ }
+
+ podNamespace, found := os.LookupEnv(PodNamespaceEnvVar)
+ if !found {
+ shouldAddOwnerRef = false
+ podNamespace = defaultNamespace
+ }
+
+ mutateConfig, err := webhookObj.CreateMutationWebhookConfiguration(podNamespace)
+ if err != nil {
+ return err
+ }
+
+ if shouldAddOwnerRef {
+ p, err := kubeClient.CoreV1().Pods(podNamespace).Get(ctx, podName, v1.GetOptions{})
+ if err != nil {
+ logger.Infof(ctx, "Failed to get Pod [%v/%v]. Error: %v", podNamespace, podName, err)
+ return fmt.Errorf("failed to get pod: %w", err)
+ }
+ mutateConfig.OwnerReferences = p.OwnerReferences
+ }
+
+ logger.Infof(ctx, "Creating MutatingWebhookConfiguration [%v/%v]", mutateConfig.GetNamespace(), mutateConfig.GetName())
+
+ _, err = kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(ctx, mutateConfig, v1.CreateOptions{})
+ var statusErr *errors.StatusError
+ if err != nil && errors2.As(err, &statusErr) && statusErr.Status().Reason == v1.StatusReasonAlreadyExists {
+ logger.Infof(ctx, "MutatingWebhookConfiguration already exists. Attempting update.")
+ obj, getErr := kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(ctx, mutateConfig.Name, v1.GetOptions{})
+ if getErr != nil {
+ return err
+ }
+ obj.Webhooks = mutateConfig.Webhooks
+ _, err = kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(ctx, obj, v1.UpdateOptions{})
+ if err == nil {
+ logger.Infof(ctx, "Successfully updated MutatingWebhookConfiguration.")
+ }
+ return err
+ } else if err != nil {
+ return fmt.Errorf("failed to create MutatingWebhookConfiguration: %w", err)
+ }
+
+ return nil
+}
diff --git a/executor/pkg/webhook/init_cert.go b/executor/pkg/webhook/init_cert.go
new file mode 100644
index 00000000000..269c04528bc
--- /dev/null
+++ b/executor/pkg/webhook/init_cert.go
@@ -0,0 +1,187 @@
+package webhook
+
+import (
+ "bytes"
+ "context"
+ cryptorand "crypto/rand"
+ "crypto/rsa"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "encoding/pem"
+ "fmt"
+ "math/big"
+ "os"
+ "path"
+ "time"
+
+ corev1 "k8s.io/api/core/v1"
+ kubeErrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes"
+ v1 "k8s.io/client-go/kubernetes/typed/core/v1"
+
+ webhookConfig "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/secret/config"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+)
+
+type webhookCerts struct {
+ CaPEM *bytes.Buffer
+ ServerPEM *bytes.Buffer
+ PrivateKeyPEM *bytes.Buffer
+}
+
+const (
+ CaCertKey = "ca.crt"
+ ServerCertKey = "tls.crt"
+ ServerCertPrivateKey = "tls.key"
+ podDefaultNamespace = "flyte"
+ permission = 0644
+ folderPerm = 0755
+)
+
+// InitCerts generates a self-signed TLS certificate for the webhook and stores it in a k8s Secret.
+func InitCerts(ctx context.Context, kubeClient kubernetes.Interface, cfg *webhookConfig.Config) error {
+ podNamespace, found := os.LookupEnv(PodNamespaceEnvVar)
+ if !found {
+ podNamespace = podDefaultNamespace
+ }
+
+ logger.Infof(ctx, "Issuing certs")
+ certs, err := createCerts(cfg.ServiceName, podNamespace)
+ if err != nil {
+ return err
+ }
+
+ logger.Infof(ctx, "Creating secret [%v] in Namespace [%v]", cfg.SecretName, podNamespace)
+ return createWebhookSecret(ctx, podNamespace, cfg, certs, kubeClient.CoreV1().Secrets(podNamespace))
+}
+
+func createWebhookSecret(ctx context.Context, namespace string, cfg *webhookConfig.Config, certs webhookCerts, secretsClient v1.SecretInterface) error {
+ isImmutable := true
+ secretData := map[string][]byte{
+ CaCertKey: certs.CaPEM.Bytes(),
+ ServerCertKey: certs.ServerPEM.Bytes(),
+ ServerCertPrivateKey: certs.PrivateKeyPEM.Bytes(),
+ }
+
+ // TODO(alex): This LocalCert tag is only for flyte running in single binary mode.
+ // In full deployment the webhook should be running in a single pod and an init container will generate and inject the secret data
+ if cfg.LocalCert {
+ certPath := cfg.ExpandCertDir()
+ if err := os.MkdirAll(certPath, folderPerm); err != nil {
+ return err
+ }
+
+ if err := os.WriteFile(path.Join(certPath, CaCertKey), certs.CaPEM.Bytes(), permission); err != nil {
+ return err
+ }
+ if err := os.WriteFile(path.Join(certPath, ServerCertKey), certs.ServerPEM.Bytes(), permission); err != nil {
+ return err
+ }
+ if err := os.WriteFile(path.Join(certPath, ServerCertPrivateKey), certs.PrivateKeyPEM.Bytes(), permission); err != nil {
+ return err
+ }
+ }
+
+ secret := &corev1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: cfg.SecretName,
+ Namespace: namespace,
+ },
+ Type: corev1.SecretTypeOpaque,
+ Data: secretData,
+ Immutable: &isImmutable,
+ }
+
+ _, err := secretsClient.Create(ctx, secret, metav1.CreateOptions{})
+ if err == nil {
+ logger.Infof(ctx, "Created secret [%v]", cfg.SecretName)
+ return nil
+ }
+
+ if kubeErrors.IsAlreadyExists(err) {
+ logger.Infof(ctx, "Secret [%v] already exists, recreating with new certs.", cfg.SecretName)
+ if err := secretsClient.Delete(ctx, cfg.SecretName, metav1.DeleteOptions{}); err != nil {
+ return err
+ }
+ if _, err := secretsClient.Create(ctx, secret, metav1.CreateOptions{}); err != nil {
+ return err
+ }
+ logger.Infof(ctx, "Recreated secret [%v]", cfg.SecretName)
+ return nil
+ }
+
+ return err
+}
+
+func createCerts(serviceName string, serviceNamespace string) (certs webhookCerts, err error) {
+ caRequest := &x509.Certificate{
+ SerialNumber: big.NewInt(2021),
+ Subject: pkix.Name{Organization: []string{"flyte.org"}},
+ NotBefore: time.Now(),
+ NotAfter: time.Now().AddDate(99, 0, 0),
+ IsCA: true,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+ KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+ BasicConstraintsValid: true,
+ }
+
+ caPrivateKey, err := rsa.GenerateKey(cryptorand.Reader, 4096)
+ if err != nil {
+ return webhookCerts{}, err
+ }
+
+ caCert, err := x509.CreateCertificate(cryptorand.Reader, caRequest, caRequest, &caPrivateKey.PublicKey, caPrivateKey)
+ if err != nil {
+ return webhookCerts{}, err
+ }
+
+ caPEM := new(bytes.Buffer)
+ if err = pem.Encode(caPEM, &pem.Block{Type: "CERTIFICATE", Bytes: caCert}); err != nil {
+ return webhookCerts{}, err
+ }
+
+ dnsNames := []string{
+ serviceName,
+ serviceName + "." + serviceNamespace,
+ serviceName + "." + serviceNamespace + ".svc",
+ }
+ commonName := serviceName + "." + serviceNamespace + ".svc"
+
+ certRequest := &x509.Certificate{
+ DNSNames: dnsNames,
+ SerialNumber: big.NewInt(1658),
+ Subject: pkix.Name{CommonName: commonName, Organization: []string{"flyte.org"}},
+ NotBefore: time.Now(),
+ NotAfter: time.Now().AddDate(99, 0, 0),
+ SubjectKeyId: []byte{1, 2, 3, 4, 6},
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+ KeyUsage: x509.KeyUsageDigitalSignature,
+ }
+
+ serverPrivateKey, err := rsa.GenerateKey(cryptorand.Reader, 4096)
+ if err != nil {
+ return webhookCerts{}, err
+ }
+
+ cert, err := x509.CreateCertificate(cryptorand.Reader, certRequest, caRequest, &serverPrivateKey.PublicKey, caPrivateKey)
+ if err != nil {
+ return webhookCerts{}, err
+ }
+
+ serverCertPEM := new(bytes.Buffer)
+ if err = pem.Encode(serverCertPEM, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil {
+ return webhookCerts{}, fmt.Errorf("failed to encode CertPEM: %w", err)
+ }
+
+ serverPrivKeyPEM := new(bytes.Buffer)
+ if err = pem.Encode(serverPrivKeyPEM, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(serverPrivateKey)}); err != nil {
+ return webhookCerts{}, fmt.Errorf("failed to encode cert private key: %w", err)
+ }
+
+ return webhookCerts{
+ CaPEM: caPEM,
+ ServerPEM: serverCertPEM,
+ PrivateKeyPEM: serverPrivKeyPEM,
+ }, nil
+}
diff --git a/executor/pkg/webhook/pod.go b/executor/pkg/webhook/pod.go
new file mode 100644
index 00000000000..7a60ecb4f39
--- /dev/null
+++ b/executor/pkg/webhook/pod.go
@@ -0,0 +1,153 @@
+// Package webhook contains PodMutator. It's a controller-runtime webhook that intercepts Pod Creation events and
+// mutates them. The SecretsMutator injects secret references into pods that have the inject-flyte-secrets label.
+package webhook
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+
+ admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+ "sigs.k8s.io/controller-runtime/pkg/manager"
+ "sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/flytek8s"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/secret"
+ webhookConfig "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/secret/config"
+ secretUtils "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/utils/secrets"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+)
+
+const webhookName = "flyte-pod-webhook.flyte.org"
+
+// PodMutator implements controller-runtime WebHook interface.
+type PodMutator struct {
+ decoder admission.Decoder
+ cfg *webhookConfig.Config
+ secretsMutator *secret.SecretsPodMutator
+}
+
+func (pm PodMutator) Handle(ctx context.Context, request admission.Request) admission.Response {
+ obj := &corev1.Pod{}
+ err := pm.decoder.Decode(request, obj)
+ if err != nil {
+ return admission.Errored(http.StatusBadRequest, err)
+ }
+
+ newObj, changed, admissionErr := pm.secretsMutator.Mutate(ctx, obj)
+ if admissionErr != nil {
+ return *admissionErr
+ }
+
+ if changed {
+ marshalled, err := json.Marshal(newObj)
+ if err != nil {
+ return admission.Errored(http.StatusInternalServerError, err)
+ }
+ return admission.PatchResponseFromRaw(request.Object.Raw, marshalled)
+ }
+
+ return admission.Allowed("No changes")
+}
+
+func (pm PodMutator) Register(ctx context.Context, mgr manager.Manager) error {
+ wh := &admission.Webhook{Handler: pm}
+ mutatePath := getPodMutatePath()
+ logger.Infof(ctx, "Registering path [%v]", mutatePath)
+ mgr.GetWebhookServer().Register(mutatePath, wh)
+ return nil
+}
+
+func (pm PodMutator) GetMutatePath() string {
+ return getPodMutatePath()
+}
+
+func getPodMutatePath() string {
+ pod := flytek8s.BuildIdentityPod()
+ return generateMutatePath(pod.GroupVersionKind())
+}
+
+func generateMutatePath(gvk schema.GroupVersionKind) string {
+ return "/mutate-" + strings.Replace(gvk.Group, ".", "-", -1) + "-" +
+ gvk.Version + "-" + strings.ToLower(gvk.Kind)
+}
+
+func (pm PodMutator) CreateMutationWebhookConfiguration(namespace string) (*admissionregistrationv1.MutatingWebhookConfiguration, error) {
+ caBytes, err := os.ReadFile(filepath.Join(pm.cfg.ExpandCertDir(), "ca.crt"))
+ if err != nil {
+ if os.IsNotExist(err) {
+ caBytes = make([]byte, 0)
+ } else {
+ return nil, err
+ }
+ }
+
+ path := pm.GetMutatePath()
+ fail := admissionregistrationv1.Fail
+ sideEffects := admissionregistrationv1.SideEffectClassNoneOnDryRun
+
+ mutateConfig := &admissionregistrationv1.MutatingWebhookConfiguration{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: pm.cfg.ServiceName,
+ Namespace: namespace,
+ },
+ Webhooks: []admissionregistrationv1.MutatingWebhook{
+ {
+ Name: webhookName,
+ ClientConfig: admissionregistrationv1.WebhookClientConfig{
+ CABundle: caBytes,
+ Service: &admissionregistrationv1.ServiceReference{
+ Name: pm.cfg.ServiceName,
+ Namespace: namespace,
+ Path: &path,
+ Port: &pm.cfg.ServicePort,
+ },
+ },
+ Rules: []admissionregistrationv1.RuleWithOperations{
+ {
+ Operations: []admissionregistrationv1.OperationType{
+ admissionregistrationv1.Create,
+ },
+ Rule: admissionregistrationv1.Rule{
+ APIGroups: []string{"*"},
+ APIVersions: []string{"v1"},
+ Resources: []string{"pods"},
+ },
+ },
+ },
+ FailurePolicy: &fail,
+ SideEffects: &sideEffects,
+ AdmissionReviewVersions: []string{"v1", "v1beta1"},
+ ObjectSelector: &metav1.LabelSelector{
+ MatchLabels: map[string]string{
+ secretUtils.PodLabel: secretUtils.PodLabelValue,
+ },
+ },
+ },
+ },
+ }
+
+ return mutateConfig, nil
+}
+
+func NewPodMutator(ctx context.Context, cfg *webhookConfig.Config, podNamespace string, scheme *runtime.Scheme, scope promutils.Scope) (*PodMutator, error) {
+ secretsMutator, err := secret.NewSecretsMutator(ctx, cfg, podNamespace, scope.NewSubScope("secrets"))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create secrets mutator: %w", err)
+ }
+
+ return &PodMutator{
+ decoder: *admission.NewDecoder(scheme),
+ cfg: cfg,
+ secretsMutator: secretsMutator,
+ }, nil
+}
diff --git a/executor/setup.go b/executor/setup.go
new file mode 100644
index 00000000000..78f5889ae59
--- /dev/null
+++ b/executor/setup.go
@@ -0,0 +1,181 @@
+package executor
+
+import (
+ "context"
+ "crypto/tls"
+ "fmt"
+ "net/http"
+ "os"
+
+ "k8s.io/apimachinery/pkg/runtime"
+ utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+ "k8s.io/client-go/kubernetes"
+ clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+ ctrl "sigs.k8s.io/controller-runtime"
+ "sigs.k8s.io/controller-runtime/pkg/healthz"
+ "sigs.k8s.io/controller-runtime/pkg/log/zap"
+ "sigs.k8s.io/controller-runtime/pkg/metrics/filters"
+ metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+ "sigs.k8s.io/controller-runtime/pkg/webhook"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/app"
+ flyteorgv1 "github.com/flyteorg/flyte/v2/executor/api/v1"
+ "github.com/flyteorg/flyte/v2/executor/pkg/config"
+ "github.com/flyteorg/flyte/v2/executor/pkg/controller"
+ "github.com/flyteorg/flyte/v2/executor/pkg/plugin"
+ webhookPkg "github.com/flyteorg/flyte/v2/executor/pkg/webhook"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery"
+ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/catalog"
+ cachecatalog "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/catalog/cache_service"
+ webhookConfig "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/pluginmachinery/secret/config"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/workflow/workflowconnect"
+
+ // Plugin registrations — blank imports trigger init() which registers
+ // plugins with the global registry.
+ _ "github.com/flyteorg/flyte/v2/flyteplugins/go/tasks/plugins/k8s/pod"
+)
+
+var scheme = runtime.NewScheme()
+
+func init() {
+ utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+ utilruntime.Must(flyteorgv1.AddToScheme(scheme))
+}
+
+// Scheme returns the runtime.Scheme with executor CRDs registered.
+// Useful for callers that need to pass the scheme to InitKubernetesClient.
+func Scheme() *runtime.Scheme {
+ return scheme
+}
+
+// Setup registers the executor as a background worker on the SetupContext.
+// Requires sc.K8sConfig and sc.DataStore to be set.
+func Setup(ctx context.Context, sc *app.SetupContext) error {
+ ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
+
+ cfg := config.GetConfig()
+
+ var tlsOpts []func(*tls.Config)
+ if !cfg.EnableHTTP2 {
+ tlsOpts = append(tlsOpts, func(c *tls.Config) {
+ c.NextProtos = []string{"http/1.1"}
+ })
+ }
+
+ wCfg := webhookConfig.GetConfig()
+ webhookServerOptions := webhook.Options{TLSOpts: tlsOpts}
+ webhookServerOptions.CertDir = wCfg.ExpandCertDir()
+ webhookServerOptions.CertName = webhookPkg.ServerCertKey
+ webhookServerOptions.KeyName = webhookPkg.ServerCertPrivateKey
+ webhookServerOptions.Port = wCfg.ListenPort
+
+ metricsServerOptions := metricsserver.Options{
+ BindAddress: cfg.MetricsBindAddress,
+ SecureServing: cfg.MetricsSecure,
+ TLSOpts: tlsOpts,
+ }
+ if cfg.MetricsSecure {
+ metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization
+ }
+ if len(cfg.MetricsCertPath) > 0 {
+ metricsServerOptions.CertDir = cfg.MetricsCertPath
+ metricsServerOptions.CertName = cfg.MetricsCertName
+ metricsServerOptions.KeyName = cfg.MetricsCertKey
+ }
+
+ mgr, err := ctrl.NewManager(sc.K8sConfig, ctrl.Options{
+ Scheme: scheme,
+ Metrics: metricsServerOptions,
+ WebhookServer: webhook.NewServer(webhookServerOptions),
+ HealthProbeBindAddress: cfg.HealthProbeBindAddress,
+ LeaderElection: cfg.LeaderElect,
+ LeaderElectionID: "abf369a8.flyte.org",
+ })
+ if err != nil {
+ return fmt.Errorf("executor: failed to create controller manager: %w", err)
+ }
+ sc.K8sCache = mgr.GetCache()
+
+ kubeClient, err := kubernetes.NewForConfig(sc.K8sConfig)
+ if err != nil {
+ return fmt.Errorf("executor: failed to create kubernetes client for webhook: %w", err)
+ }
+
+ podNamespace := os.Getenv(webhookPkg.PodNamespaceEnvVar)
+ if podNamespace == "" {
+ podNamespace = sc.Namespace
+ }
+
+ if err := webhookPkg.Setup(ctx, kubeClient, wCfg, podNamespace, promutils.NewScope("executor"), mgr); err != nil {
+ return fmt.Errorf("executor: webhook setup failed: %w", err)
+ }
+
+ dataStore, err := storage.NewDataStore(storage.GetConfig(), promutils.NewScope("executor:storage"))
+ if err != nil {
+ return fmt.Errorf("executor: failed to create data store: %w", err)
+ }
+
+ setupCtx := plugin.NewSetupContext(
+ mgr, nil, nil, nil, nil,
+ "TaskAction",
+ promutils.NewScope("executor"),
+ )
+ registry := plugin.NewRegistry(setupCtx, pluginmachinery.PluginRegistry())
+ if err := registry.Initialize(ctx); err != nil {
+ return fmt.Errorf("executor: failed to initialize plugin registry: %w", err)
+ }
+
+ eventsServiceURL := sc.BaseURL
+ if eventsServiceURL == "" {
+ eventsServiceURL = cfg.EventsServiceURL
+ }
+ eventsClient := workflowconnect.NewEventsProxyServiceClient(http.DefaultClient, eventsServiceURL)
+ catalogCfg := catalog.GetConfig()
+ cacheServiceURL := sc.BaseURL
+ if cacheServiceURL == "" {
+ cacheServiceURL = cfg.CacheServiceURL
+ }
+ cacheClient := cachecatalog.NewHTTPClient(dataStore, cacheServiceURL, catalogCfg.MaxCacheAge.Duration)
+ asyncCatalogClient, err := catalog.NewAsyncClient(cacheClient, *catalogCfg, promutils.NewScope("executor:catalog"))
+ if err != nil {
+ return fmt.Errorf("executor: failed to create catalog cache client: %w", err)
+ }
+ if err := asyncCatalogClient.Start(ctx); err != nil {
+ return fmt.Errorf("executor: failed to start catalog cache client: %w", err)
+ }
+
+ reconciler := controller.NewTaskActionReconciler(
+ mgr.GetClient(), mgr.GetScheme(), registry, dataStore, eventsClient, cfg.Cluster,
+ )
+ reconciler.CatalogClient = asyncCatalogClient
+ reconciler.Catalog = cacheClient
+ reconciler.Recorder = mgr.GetEventRecorderFor("taskaction-controller")
+ if err := reconciler.SetupWithManager(mgr); err != nil {
+ return fmt.Errorf("executor: failed to setup controller: %w", err)
+ }
+
+ if cfg.GC.Interval.Duration > 0 {
+ if cfg.GC.MaxTTL.Duration <= 0 {
+ return fmt.Errorf("executor: gc.maxTTL must be positive when gc is enabled, got %v", cfg.GC.MaxTTL.Duration)
+ }
+ gc := controller.NewGarbageCollector(mgr.GetClient(), cfg.GC.Interval.Duration, cfg.GC.MaxTTL.Duration)
+ if err := mgr.Add(gc); err != nil {
+ return fmt.Errorf("executor: failed to add garbage collector: %w", err)
+ }
+ }
+
+ if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+ return fmt.Errorf("executor: failed to add health check: %w", err)
+ }
+ if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+ return fmt.Errorf("executor: failed to add ready check: %w", err)
+ }
+
+ sc.AddWorker("executor", func(ctx context.Context) error {
+ return mgr.Start(ctx)
+ })
+
+ return nil
+}
diff --git a/executor/test/e2e/e2e_suite_test.go b/executor/test/e2e/e2e_suite_test.go
new file mode 100644
index 00000000000..d6b44cd10fc
--- /dev/null
+++ b/executor/test/e2e/e2e_suite_test.go
@@ -0,0 +1,92 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+ "testing"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+
+ "github.com/flyteorg/flyte/v2/executor/test/utils"
+)
+
+var (
+ // Optional Environment Variables:
+ // - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup.
+ // These variables are useful if CertManager is already installed, avoiding
+ // re-installation and conflicts.
+ skipCertManagerInstall = os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true"
+ // isCertManagerAlreadyInstalled will be set true when CertManager CRDs be found on the cluster
+ isCertManagerAlreadyInstalled = false
+
+ // projectImage is the name of the image which will be build and loaded
+ // with the code source changes to be tested.
+ projectImage = "example.com/executor:v0.0.1"
+)
+
+// TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated,
+// temporary environment to validate project changes with the purpose of being used in CI jobs.
+// The default setup requires Kind, builds/loads the Manager Docker image locally, and installs
+// CertManager.
+func TestE2E(t *testing.T) {
+ RegisterFailHandler(Fail)
+ _, _ = fmt.Fprintf(GinkgoWriter, "Starting executor integration test suite\n")
+ RunSpecs(t, "e2e suite")
+}
+
+var _ = BeforeSuite(func() {
+ By("building the manager(Operator) image")
+ cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage))
+ _, err := utils.Run(cmd)
+ ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image")
+
+ // TODO(user): If you want to change the e2e test vendor from Kind, ensure the image is
+ // built and available before running the tests. Also, remove the following block.
+ By("loading the manager(Operator) image on Kind")
+ err = utils.LoadImageToKindClusterWithName(projectImage)
+ ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind")
+
+ // The tests-e2e are intended to run on a temporary cluster that is created and destroyed for testing.
+ // To prevent errors when tests run in environments with CertManager already installed,
+ // we check for its presence before execution.
+ // Setup CertManager before the suite if not skipped and if not already installed
+ if !skipCertManagerInstall {
+ By("checking if cert manager is installed already")
+ isCertManagerAlreadyInstalled = utils.IsCertManagerCRDsInstalled()
+ if !isCertManagerAlreadyInstalled {
+ _, _ = fmt.Fprintf(GinkgoWriter, "Installing CertManager...\n")
+ Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager")
+ } else {
+ _, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n")
+ }
+ }
+})
+
+var _ = AfterSuite(func() {
+ // Teardown CertManager after the suite if not skipped and if it was not already installed
+ if !skipCertManagerInstall && !isCertManagerAlreadyInstalled {
+ _, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n")
+ utils.UninstallCertManager()
+ }
+})
diff --git a/executor/test/e2e/e2e_test.go b/executor/test/e2e/e2e_test.go
new file mode 100644
index 00000000000..37b87a1f427
--- /dev/null
+++ b/executor/test/e2e/e2e_test.go
@@ -0,0 +1,334 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "time"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+
+ "github.com/flyteorg/flyte/v2/executor/test/utils"
+)
+
+// namespace where the project is deployed in
+const namespace = "executor-system"
+
+// serviceAccountName created for the project
+const serviceAccountName = "executor-controller-manager"
+
+// metricsServiceName is the name of the metrics service of the project
+const metricsServiceName = "executor-controller-manager-metrics-service"
+
+// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data
+const metricsRoleBindingName = "executor-metrics-binding"
+
+var _ = Describe("Manager", Ordered, func() {
+ var controllerPodName string
+
+ // Before running the tests, set up the environment by creating the namespace,
+ // enforce the restricted security policy to the namespace, installing CRDs,
+ // and deploying the controller.
+ BeforeAll(func() {
+ By("creating manager namespace")
+ cmd := exec.Command("kubectl", "create", "ns", namespace)
+ _, err := utils.Run(cmd)
+ Expect(err).NotTo(HaveOccurred(), "Failed to create namespace")
+
+ By("labeling the namespace to enforce the restricted security policy")
+ cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace,
+ "pod-security.kubernetes.io/enforce=restricted")
+ _, err = utils.Run(cmd)
+ Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy")
+
+ By("installing CRDs")
+ cmd = exec.Command("make", "install")
+ _, err = utils.Run(cmd)
+ Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs")
+
+ By("deploying the controller-manager")
+ cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage))
+ _, err = utils.Run(cmd)
+ Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager")
+ })
+
+ // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs,
+ // and deleting the namespace.
+ AfterAll(func() {
+ By("cleaning up the curl pod for metrics")
+ cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace)
+ _, _ = utils.Run(cmd)
+
+ By("undeploying the controller-manager")
+ cmd = exec.Command("make", "undeploy")
+ _, _ = utils.Run(cmd)
+
+ By("uninstalling CRDs")
+ cmd = exec.Command("make", "uninstall")
+ _, _ = utils.Run(cmd)
+
+ By("removing manager namespace")
+ cmd = exec.Command("kubectl", "delete", "ns", namespace)
+ _, _ = utils.Run(cmd)
+ })
+
+ // After each test, check for failures and collect logs, events,
+ // and pod descriptions for debugging.
+ AfterEach(func() {
+ specReport := CurrentSpecReport()
+ if specReport.Failed() {
+ By("Fetching controller manager pod logs")
+ cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
+ controllerLogs, err := utils.Run(cmd)
+ if err == nil {
+ _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs)
+ } else {
+ _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err)
+ }
+
+ By("Fetching Kubernetes events")
+ cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp")
+ eventsOutput, err := utils.Run(cmd)
+ if err == nil {
+ _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput)
+ } else {
+ _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err)
+ }
+
+ By("Fetching curl-metrics logs")
+ cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace)
+ metricsOutput, err := utils.Run(cmd)
+ if err == nil {
+ _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput)
+ } else {
+ _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err)
+ }
+
+ By("Fetching controller manager pod description")
+ cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace)
+ podDescription, err := utils.Run(cmd)
+ if err == nil {
+ fmt.Println("Pod description:\n", podDescription)
+ } else {
+ fmt.Println("Failed to describe controller pod")
+ }
+ }
+ })
+
+ SetDefaultEventuallyTimeout(2 * time.Minute)
+ SetDefaultEventuallyPollingInterval(time.Second)
+
+ Context("Manager", func() {
+ It("should run successfully", func() {
+ By("validating that the controller-manager pod is running as expected")
+ verifyControllerUp := func(g Gomega) {
+ // Get the name of the controller-manager pod
+ cmd := exec.Command("kubectl", "get",
+ "pods", "-l", "control-plane=controller-manager",
+ "-o", "go-template={{ range .items }}"+
+ "{{ if not .metadata.deletionTimestamp }}"+
+ "{{ .metadata.name }}"+
+ "{{ \"\\n\" }}{{ end }}{{ end }}",
+ "-n", namespace,
+ )
+
+ podOutput, err := utils.Run(cmd)
+ g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information")
+ podNames := utils.GetNonEmptyLines(podOutput)
+ g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running")
+ controllerPodName = podNames[0]
+ g.Expect(controllerPodName).To(ContainSubstring("controller-manager"))
+
+ // Validate the pod's status
+ cmd = exec.Command("kubectl", "get",
+ "pods", controllerPodName, "-o", "jsonpath={.status.phase}",
+ "-n", namespace,
+ )
+ output, err := utils.Run(cmd)
+ g.Expect(err).NotTo(HaveOccurred())
+ g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status")
+ }
+ Eventually(verifyControllerUp).Should(Succeed())
+ })
+
+ It("should ensure the metrics endpoint is serving metrics", func() {
+ By("creating a ClusterRoleBinding for the service account to allow access to metrics")
+ cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName,
+ "--clusterrole=executor-metrics-reader",
+ fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName),
+ )
+ _, err := utils.Run(cmd)
+ Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding")
+
+ By("validating that the metrics service is available")
+ cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace)
+ _, err = utils.Run(cmd)
+ Expect(err).NotTo(HaveOccurred(), "Metrics service should exist")
+
+ By("getting the service account token")
+ token, err := serviceAccountToken()
+ Expect(err).NotTo(HaveOccurred())
+ Expect(token).NotTo(BeEmpty())
+
+ By("waiting for the metrics endpoint to be ready")
+ verifyMetricsEndpointReady := func(g Gomega) {
+ cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace)
+ output, err := utils.Run(cmd)
+ g.Expect(err).NotTo(HaveOccurred())
+ g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready")
+ }
+ Eventually(verifyMetricsEndpointReady).Should(Succeed())
+
+ By("verifying that the controller manager is serving the metrics server")
+ verifyMetricsServerStarted := func(g Gomega) {
+ cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
+ output, err := utils.Run(cmd)
+ g.Expect(err).NotTo(HaveOccurred())
+ g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"),
+ "Metrics server not yet started")
+ }
+ Eventually(verifyMetricsServerStarted).Should(Succeed())
+
+ By("creating the curl-metrics pod to access the metrics endpoint")
+ cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
+ "--namespace", namespace,
+ "--image=curlimages/curl:latest",
+ "--overrides",
+ fmt.Sprintf(`{
+ "spec": {
+ "containers": [{
+ "name": "curl",
+ "image": "curlimages/curl:latest",
+ "command": ["/bin/sh", "-c"],
+ "args": ["curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics"],
+ "securityContext": {
+ "readOnlyRootFilesystem": true,
+ "allowPrivilegeEscalation": false,
+ "capabilities": {
+ "drop": ["ALL"]
+ },
+ "runAsNonRoot": true,
+ "runAsUser": 1000,
+ "seccompProfile": {
+ "type": "RuntimeDefault"
+ }
+ }
+ }],
+ "serviceAccountName": "%s"
+ }
+ }`, token, metricsServiceName, namespace, serviceAccountName))
+ _, err = utils.Run(cmd)
+ Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod")
+
+ By("waiting for the curl-metrics pod to complete.")
+ verifyCurlUp := func(g Gomega) {
+ cmd := exec.Command("kubectl", "get", "pods", "curl-metrics",
+ "-o", "jsonpath={.status.phase}",
+ "-n", namespace)
+ output, err := utils.Run(cmd)
+ g.Expect(err).NotTo(HaveOccurred())
+ g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status")
+ }
+ Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed())
+
+ By("getting the metrics by checking curl-metrics logs")
+ verifyMetricsAvailable := func(g Gomega) {
+ metricsOutput, err := getMetricsOutput()
+ g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod")
+ g.Expect(metricsOutput).NotTo(BeEmpty())
+ g.Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK"))
+ }
+ Eventually(verifyMetricsAvailable, 2*time.Minute).Should(Succeed())
+ })
+
+ // +kubebuilder:scaffold:e2e-webhooks-checks
+
+ // TODO: Customize the e2e test suite with scenarios specific to your project.
+ // Consider applying sample/CR(s) and check their status and/or verifying
+ // the reconciliation by using the metrics, i.e.:
+ // metricsOutput, err := getMetricsOutput()
+ // Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod")
+ // Expect(metricsOutput).To(ContainSubstring(
+ // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`,
+ // strings.ToLower(),
+ // ))
+ })
+})
+
+// serviceAccountToken returns a token for the specified service account in the given namespace.
+// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request
+// and parsing the resulting token from the API response.
+func serviceAccountToken() (string, error) {
+ const tokenRequestRawString = `{
+ "apiVersion": "authentication.k8s.io/v1",
+ "kind": "TokenRequest"
+ }`
+
+ // Temporary file to store the token request
+ secretName := fmt.Sprintf("%s-token-request", serviceAccountName)
+ tokenRequestFile := filepath.Join("/tmp", secretName)
+ err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644))
+ if err != nil {
+ return "", err
+ }
+
+ var out string
+ verifyTokenCreation := func(g Gomega) {
+ // Execute kubectl command to create the token
+ cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf(
+ "/api/v1/namespaces/%s/serviceaccounts/%s/token",
+ namespace,
+ serviceAccountName,
+ ), "-f", tokenRequestFile)
+
+ output, err := cmd.CombinedOutput()
+ g.Expect(err).NotTo(HaveOccurred())
+
+ // Parse the JSON output to extract the token
+ var token tokenRequest
+ err = json.Unmarshal(output, &token)
+ g.Expect(err).NotTo(HaveOccurred())
+
+ out = token.Status.Token
+ }
+ Eventually(verifyTokenCreation).Should(Succeed())
+
+ return out, err
+}
+
+// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint.
+func getMetricsOutput() (string, error) {
+ By("getting the curl-metrics logs")
+ cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace)
+ return utils.Run(cmd)
+}
+
+// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response,
+// containing only the token field that we need to extract.
+type tokenRequest struct {
+ Status struct {
+ Token string `json:"token"`
+ } `json:"status"`
+}
diff --git a/executor/test/utils/utils.go b/executor/test/utils/utils.go
new file mode 100644
index 00000000000..cf67d9019a2
--- /dev/null
+++ b/executor/test/utils/utils.go
@@ -0,0 +1,226 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+ "bufio"
+ "bytes"
+ "fmt"
+ "os"
+ "os/exec"
+ "strings"
+
+ . "github.com/onsi/ginkgo/v2" // nolint:revive,staticcheck
+)
+
+const (
+ certmanagerVersion = "v1.18.2"
+ certmanagerURLTmpl = "https://github.com/cert-manager/cert-manager/releases/download/%s/cert-manager.yaml"
+
+ defaultKindBinary = "kind"
+ defaultKindCluster = "kind"
+)
+
+func warnError(err error) {
+ _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err)
+}
+
+// Run executes the provided command within this context
+func Run(cmd *exec.Cmd) (string, error) {
+ dir, _ := GetProjectDir()
+ cmd.Dir = dir
+
+ if err := os.Chdir(cmd.Dir); err != nil {
+ _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %q\n", err)
+ }
+
+ cmd.Env = append(os.Environ(), "GO111MODULE=on")
+ command := strings.Join(cmd.Args, " ")
+ _, _ = fmt.Fprintf(GinkgoWriter, "running: %q\n", command)
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return string(output), fmt.Errorf("%q failed with error %q: %w", command, string(output), err)
+ }
+
+ return string(output), nil
+}
+
+// UninstallCertManager uninstalls the cert manager
+func UninstallCertManager() {
+ url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion)
+ cmd := exec.Command("kubectl", "delete", "-f", url)
+ if _, err := Run(cmd); err != nil {
+ warnError(err)
+ }
+
+ // Delete leftover leases in kube-system (not cleaned by default)
+ kubeSystemLeases := []string{
+ "cert-manager-cainjector-leader-election",
+ "cert-manager-controller",
+ }
+ for _, lease := range kubeSystemLeases {
+ cmd = exec.Command("kubectl", "delete", "lease", lease,
+ "-n", "kube-system", "--ignore-not-found", "--force", "--grace-period=0")
+ if _, err := Run(cmd); err != nil {
+ warnError(err)
+ }
+ }
+}
+
+// InstallCertManager installs the cert manager bundle.
+func InstallCertManager() error {
+ url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion)
+ cmd := exec.Command("kubectl", "apply", "-f", url)
+ if _, err := Run(cmd); err != nil {
+ return err
+ }
+ // Wait for cert-manager-webhook to be ready, which can take time if cert-manager
+ // was re-installed after uninstalling on a cluster.
+ cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook",
+ "--for", "condition=Available",
+ "--namespace", "cert-manager",
+ "--timeout", "5m",
+ )
+
+ _, err := Run(cmd)
+ return err
+}
+
+// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed
+// by verifying the existence of key CRDs related to Cert Manager.
+func IsCertManagerCRDsInstalled() bool {
+ // List of common Cert Manager CRDs
+ certManagerCRDs := []string{
+ "certificates.cert-manager.io",
+ "issuers.cert-manager.io",
+ "clusterissuers.cert-manager.io",
+ "certificaterequests.cert-manager.io",
+ "orders.acme.cert-manager.io",
+ "challenges.acme.cert-manager.io",
+ }
+
+ // Execute the kubectl command to get all CRDs
+ cmd := exec.Command("kubectl", "get", "crds")
+ output, err := Run(cmd)
+ if err != nil {
+ return false
+ }
+
+ // Check if any of the Cert Manager CRDs are present
+ crdList := GetNonEmptyLines(output)
+ for _, crd := range certManagerCRDs {
+ for _, line := range crdList {
+ if strings.Contains(line, crd) {
+ return true
+ }
+ }
+ }
+
+ return false
+}
+
+// LoadImageToKindClusterWithName loads a local docker image to the kind cluster
+func LoadImageToKindClusterWithName(name string) error {
+ cluster := defaultKindCluster
+ if v, ok := os.LookupEnv("KIND_CLUSTER"); ok {
+ cluster = v
+ }
+ kindOptions := []string{"load", "docker-image", name, "--name", cluster}
+ kindBinary := defaultKindBinary
+ if v, ok := os.LookupEnv("KIND"); ok {
+ kindBinary = v
+ }
+ cmd := exec.Command(kindBinary, kindOptions...)
+ _, err := Run(cmd)
+ return err
+}
+
+// GetNonEmptyLines converts given command output string into individual objects
+// according to line breakers, and ignores the empty elements in it.
+func GetNonEmptyLines(output string) []string {
+ var res []string
+ elements := strings.Split(output, "\n")
+ for _, element := range elements {
+ if element != "" {
+ res = append(res, element)
+ }
+ }
+
+ return res
+}
+
+// GetProjectDir will return the directory where the project is
+func GetProjectDir() (string, error) {
+ wd, err := os.Getwd()
+ if err != nil {
+ return wd, fmt.Errorf("failed to get current working directory: %w", err)
+ }
+ wd = strings.ReplaceAll(wd, "/test/e2e", "")
+ return wd, nil
+}
+
+// UncommentCode searches for target in the file and remove the comment prefix
+// of the target content. The target content may span multiple lines.
+func UncommentCode(filename, target, prefix string) error {
+ // false positive
+ // nolint:gosec
+ content, err := os.ReadFile(filename)
+ if err != nil {
+ return fmt.Errorf("failed to read file %q: %w", filename, err)
+ }
+ strContent := string(content)
+
+ idx := strings.Index(strContent, target)
+ if idx < 0 {
+ return fmt.Errorf("unable to find the code %q to be uncomment", target)
+ }
+
+ out := new(bytes.Buffer)
+ _, err = out.Write(content[:idx])
+ if err != nil {
+ return fmt.Errorf("failed to write to output: %w", err)
+ }
+
+ scanner := bufio.NewScanner(bytes.NewBufferString(target))
+ if !scanner.Scan() {
+ return nil
+ }
+ for {
+ if _, err = out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)); err != nil {
+ return fmt.Errorf("failed to write to output: %w", err)
+ }
+ // Avoid writing a newline in case the previous line was the last in target.
+ if !scanner.Scan() {
+ break
+ }
+ if _, err = out.WriteString("\n"); err != nil {
+ return fmt.Errorf("failed to write to output: %w", err)
+ }
+ }
+
+ if _, err = out.Write(content[idx+len(target):]); err != nil {
+ return fmt.Errorf("failed to write to output: %w", err)
+ }
+
+ // false positive
+ // nolint:gosec
+ if err = os.WriteFile(filename, out.Bytes(), 0644); err != nil {
+ return fmt.Errorf("failed to write file %q: %w", filename, err)
+ }
+
+ return nil
+}
diff --git a/fast.tar.gz b/fast.tar.gz
new file mode 100644
index 00000000000..99dc36de6b4
Binary files /dev/null and b/fast.tar.gz differ
diff --git a/flytecopilot/.gitignore b/flytecopilot/.gitignore
new file mode 100644
index 00000000000..6f17a622f0a
--- /dev/null
+++ b/flytecopilot/.gitignore
@@ -0,0 +1,17 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+.idea
+artifacts/*
diff --git a/flytecopilot/.golangci.yml b/flytecopilot/.golangci.yml
new file mode 100644
index 00000000000..71a85ec5c36
--- /dev/null
+++ b/flytecopilot/.golangci.yml
@@ -0,0 +1,32 @@
+run:
+ skip-dirs:
+ - pkg/client
+linters:
+ disable-all: true
+ enable:
+ - errcheck
+ - gosec
+ - gci
+ - goconst
+ - goimports
+ - gosimple
+ - govet
+ - ineffassign
+ - misspell
+ - nakedret
+ - staticcheck
+ - typecheck
+ - unconvert
+ - unparam
+ - unused
+ - protogetter
+linters-settings:
+ gci:
+ custom-order: true
+ sections:
+ - standard
+ - default
+ - prefix(github.com/flyteorg)
+ skip-generated: true
+ goconst:
+ ignore-tests: true
diff --git a/flytecopilot/.goreleaser.yml b/flytecopilot/.goreleaser.yml
new file mode 100644
index 00000000000..07e0f7d8d18
--- /dev/null
+++ b/flytecopilot/.goreleaser.yml
@@ -0,0 +1,36 @@
+project_name: flytecopilot
+before:
+ hooks:
+ - go mod download
+builds:
+ - id: flytecopilot
+ env:
+ - CGO_ENABLED=0
+ main: ./main.go
+ ldflags:
+ - -s -w -X github.com/flyteorg/flytestdlib/version.Version={{.Version}} -X github.com/flyteorg/flytestdlib/version.Build={{.ShortCommit}} -X github.com/flyteorg/flytestdlib/version.BuildTime={{.Date}}
+ binary: flytecopilot
+ goos:
+ - linux
+ - windows
+ - darwin
+archives:
+ - id: flytecopilot-archive
+ name_template: |-
+ flytecopilot_{{ .Tag }}_{{ .Os }}_
+ {{- if eq .Arch "amd64" }}x86_64
+ {{- else if eq .Arch "386" }}i386
+ {{- else }}{{ .Arch }}{{ end }}
+ builds:
+ - flytecopilot
+ format_overrides:
+ - goos: windows
+ format: zip
+checksum:
+ name_template: 'checksums.txt'
+changelog:
+ sort: asc
+ filters:
+ exclude:
+ - '^docs:'
+ - '^test:'
diff --git a/flytecopilot/Makefile b/flytecopilot/Makefile
new file mode 100755
index 00000000000..8a21a884ffd
--- /dev/null
+++ b/flytecopilot/Makefile
@@ -0,0 +1,27 @@
+export REPOSITORY=flytecopilot
+export REPO_ROOT=..
+include ../boilerplate/flyte/docker_build/Makefile
+include ../boilerplate/flyte/golang_test_targets/Makefile
+
+.PHONY: update_boilerplate
+update_boilerplate:
+ @curl https://raw.githubusercontent.com/flyteorg/boilerplate/master/boilerplate/update.sh -o boilerplate/update.sh
+ @boilerplate/update.sh
+
+clean:
+ rm -rf bin
+
+.PHONY: linux_compile
+linux_compile: export CGO_ENABLED ?= 0
+linux_compile: export GOOS ?= linux
+linux_compile:
+ go build -o /artifacts/flyte-copilot .
+
+.PHONY: compile
+compile:
+ mkdir -p ./artifacts
+ go build -o ./artifacts/flyte-copilot .
+
+cross_compile:
+ @mkdir -p ./artifacts/cross
+ GOOS=linux GOARCH=amd64 go build -o ./artifacts/flyte-copilot .
diff --git a/flytecopilot/README.md b/flytecopilot/README.md
new file mode 100644
index 00000000000..92304e947f8
--- /dev/null
+++ b/flytecopilot/README.md
@@ -0,0 +1,52 @@
+# Flyte CoPilot
+
+## Overview
+Flyte CoPilot provides a sidecar that understand Flyte Metadata Format as specified in FlyteIDL and make it possible to run arbitrary containers in Flyte.
+This is achieved using `flyte-copilot` a binary that runs in 2 modes,
+ -*Downloader* - Downloads the metadata and any other data (if configured) to a provided path. In kubernetes this path could be a shared volume.
+ - *Sidecar* - Monitors the process and uploads any data that is generated by the process in a prescribed path/
+
+## Mode: Downloader
+
+```bash
+$ flyte-copilot downloader
+```
+
+In K8s `flyte-copilot downloader` can be run as part of the init containers with the download volume mounted. This guarantees that the metadata and any data (if configured)
+is downloaded before the main container starts up.
+
+## Mode: Sidecar
+ As a sidecar process, that runs in parallel with the main container/process, the goal is to
+ 1. identify the main container
+ 2. Wait for the main container to start up
+ 3. Wait for the main container to exit
+ 4. Copy the data to remote store (especially the metadata)
+ 5. Exit
+
+```bash
+$ flyte-copilot sidecar
+```
+
+### Raw notes
+ Solution 1:
+ poll Kubeapi.
+ - Works perfectly fine, but too much load on kubeapi
+
+ Solution 2:
+ Create a protocol. Main container will exit and write a _SUCCESS file to a known location
+ - problem in the case of oom or random exits. Uploader will be stuck. We could use a timeout? and in the sidecar just kill the pod, when the main exits unhealthy?
+
+ Solution 3:
+ Use shared process namespace. This allows all pids in a pod to share the namespace. Thus pids can see each other.
+
+ Problems:
+ How to identify the main container?
+ - Container id is not known ahead of time and container name -> Pid mapping is not possible?
+ - How to wait for main container to start up.
+ One solution for both, call kubeapi and get pod info and find the container id
+
+ Note: we can poll /proc/pid/cgroup file (it contains the container id) so we can create a blind container id to pid mapping. Then somehow get the main container id
+
+ Once we know the main container, waiting for it to exit is simple and implemented
+ Copying data is simple and implemented
+
diff --git a/flytecopilot/cmd/containerwatcher/iface.go b/flytecopilot/cmd/containerwatcher/iface.go
new file mode 100644
index 00000000000..766d6f59887
--- /dev/null
+++ b/flytecopilot/cmd/containerwatcher/iface.go
@@ -0,0 +1,28 @@
+package containerwatcher
+
+import (
+ "context"
+ "fmt"
+)
+
+var ErrTimeout = fmt.Errorf("timeout while waiting")
+
+type Watcher interface {
+ WaitToStart(ctx context.Context) error
+ WaitToExit(ctx context.Context) error
+}
+
+type WatcherType = string
+
+const (
+ // Uses Kube 1.28 feature - https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/
+ // Watching SIGTERM when main container exit
+ WatcherTypeSignal WatcherType = "signal"
+ // Dummy watcher. Exits immediately, assuming success
+ WatcherTypeNoop WatcherType = "noop"
+)
+
+var AllWatcherTypes = []WatcherType{
+ WatcherTypeSignal,
+ WatcherTypeNoop,
+}
diff --git a/flytecopilot/cmd/containerwatcher/noop_watcher.go b/flytecopilot/cmd/containerwatcher/noop_watcher.go
new file mode 100644
index 00000000000..2084e8afcfe
--- /dev/null
+++ b/flytecopilot/cmd/containerwatcher/noop_watcher.go
@@ -0,0 +1,20 @@
+package containerwatcher
+
+import (
+ "context"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+)
+
+type NoopWatcher struct {
+}
+
+func (n NoopWatcher) WaitToStart(ctx context.Context) error {
+ logger.Warn(ctx, "noop container watcher setup. assuming container started.")
+ return nil
+}
+
+func (n NoopWatcher) WaitToExit(ctx context.Context) error {
+ logger.Warn(ctx, "noop container watcher setup. assuming container exited.")
+ return nil
+}
diff --git a/flytecopilot/cmd/containerwatcher/signal_watcher.go b/flytecopilot/cmd/containerwatcher/signal_watcher.go
new file mode 100644
index 00000000000..1d4615337ed
--- /dev/null
+++ b/flytecopilot/cmd/containerwatcher/signal_watcher.go
@@ -0,0 +1,38 @@
+package containerwatcher
+
+import (
+ "context"
+ "os"
+ "os/signal"
+ "syscall"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+)
+
+type SignalWatcher struct {
+}
+
+func (n SignalWatcher) WaitToStart(ctx context.Context) error {
+ logger.Warn(ctx, "WaitToStart is not needed for signal watcher.")
+ return nil
+}
+
+func (n SignalWatcher) WaitToExit(ctx context.Context) error {
+ logger.Infof(ctx, "Signal Watcher waiting for termination signal")
+ defer logger.Infof(ctx, "Signal Watcher exiting on termination signal")
+
+ // Listen for SIGTERM
+ sigs := make(chan os.Signal, 1)
+ signal.Notify(sigs, syscall.SIGTERM)
+ defer signal.Stop(sigs)
+
+ // Wait for SIGTERM signal or cancel context
+ select {
+ case sig := <-sigs:
+ logger.Infof(ctx, "Received signal: %v", sig)
+ return nil
+ case <-ctx.Done():
+ logger.Infof(ctx, "Context canceled")
+ return nil
+ }
+}
diff --git a/flytecopilot/cmd/download.go b/flytecopilot/cmd/download.go
new file mode 100644
index 00000000000..0a575cfe6e3
--- /dev/null
+++ b/flytecopilot/cmd/download.go
@@ -0,0 +1,121 @@
+package cmd
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/spf13/cobra"
+
+ "github.com/flyteorg/flyte/v2/flytecopilot/data"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+type DownloadOptions struct {
+ *RootOptions
+ remoteInputsPath string
+ remoteOutputsPrefix string
+ localDirectoryPath string
+ inputInterface []byte
+ metadataFormat string
+ downloadMode string
+ timeout time.Duration
+}
+
+func GetFormatVals() []string {
+ var vals []string
+ for k := range core.DataLoadingConfig_LiteralMapFormat_value {
+ vals = append(vals, k)
+ }
+ return vals
+}
+
+func GetDownloadModeVals() []string {
+ var vals []string
+ for k := range core.IOStrategy_DownloadMode_value {
+ vals = append(vals, k)
+ }
+ return vals
+}
+
+func GetUploadModeVals() []string {
+ var vals []string
+ for k := range core.IOStrategy_UploadMode_value {
+ vals = append(vals, k)
+ }
+ return vals
+}
+
+func (d *DownloadOptions) Download(ctx context.Context) error {
+ if d.remoteOutputsPrefix == "" {
+ return fmt.Errorf("to-output-prefix is required")
+ }
+
+ // We need remote outputs prefix to write and error file
+ err := func() error {
+ if d.localDirectoryPath == "" {
+ return fmt.Errorf("to-local-dir is required")
+ }
+ if d.remoteInputsPath == "" {
+ return fmt.Errorf("from-remote is required")
+ }
+ f, ok := core.DataLoadingConfig_LiteralMapFormat_value[d.metadataFormat]
+ if !ok {
+ return fmt.Errorf("incorrect input download format specified, given [%s], possible values [%+v]", d.metadataFormat, GetFormatVals())
+ }
+
+ m, ok := core.IOStrategy_DownloadMode_value[d.downloadMode]
+ if !ok {
+ return fmt.Errorf("incorrect input download mode specified, given [%s], possible values [%+v]", d.downloadMode, GetDownloadModeVals())
+ }
+ dl := data.NewDownloader(ctx, d.Store, core.DataLoadingConfig_LiteralMapFormat(f), core.IOStrategy_DownloadMode(m))
+ childCtx := ctx
+ cancelFn := func() {}
+ if d.timeout > 0 {
+ childCtx, cancelFn = context.WithTimeout(ctx, d.timeout)
+ }
+ defer cancelFn()
+ err := dl.DownloadInputs(childCtx, storage.DataReference(d.remoteInputsPath), d.localDirectoryPath)
+ if err != nil {
+ logger.Errorf(ctx, "Downloading failed, err %s", err)
+ return err
+ }
+ return nil
+ }()
+
+ if err != nil {
+ if err2 := d.UploadError(ctx, "InputDownloadFailed", err, storage.DataReference(d.remoteOutputsPrefix)); err2 != nil {
+ logger.Errorf(ctx, "Failed to write error document, err :%s", err2)
+ return err2
+ }
+ }
+ return nil
+}
+
+func NewDownloadCommand(opts *RootOptions) *cobra.Command {
+
+ downloadOpts := &DownloadOptions{
+ RootOptions: opts,
+ }
+
+ // deleteCmd represents the delete command
+ downloadCmd := &cobra.Command{
+ Use: "download ",
+ Short: "downloads flytedata from the remotepath to a local directory.",
+ Long: `Currently it looks at the outputs.pb and creates one file per variable.`,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return downloadOpts.Download(context.Background())
+ },
+ }
+
+ downloadCmd.Flags().StringVarP(&downloadOpts.remoteInputsPath, "from-remote", "f", "", "The remote path/key for inputs in stow store.")
+ downloadCmd.Flags().StringVarP(&downloadOpts.remoteOutputsPrefix, "to-output-prefix", "", "", "The remote path/key prefix for outputs in stow store. this is mostly used to write errors.pb.")
+ downloadCmd.Flags().StringVarP(&downloadOpts.localDirectoryPath, "to-local-dir", "o", "", "The local directory on disk where data should be downloaded.")
+ downloadCmd.Flags().StringVarP(&downloadOpts.metadataFormat, "format", "m", core.DataLoadingConfig_JSON.String(), fmt.Sprintf("What should be the output format for the primitive and structured types. Options [%v]", GetFormatVals()))
+ downloadCmd.Flags().StringVarP(&downloadOpts.downloadMode, "download-mode", "d", core.IOStrategy_DOWNLOAD_EAGER.String(), fmt.Sprintf("Download mode to use. Options [%v]", GetDownloadModeVals()))
+ downloadCmd.Flags().DurationVarP(&downloadOpts.timeout, "timeout", "t", time.Hour*1, "Max time to allow for downloads to complete, default is 1H")
+ downloadCmd.Flags().BytesBase64VarP(&downloadOpts.inputInterface, "input-interface", "i", nil, "Input interface proto message - core.VariableMap, base64 encoced string")
+ return downloadCmd
+}
diff --git a/flytecopilot/cmd/download_test.go b/flytecopilot/cmd/download_test.go
new file mode 100644
index 00000000000..4d3a477af9e
--- /dev/null
+++ b/flytecopilot/cmd/download_test.go
@@ -0,0 +1,189 @@
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "sort"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/flyteorg/flyte/v2/flyteidl2/clients/go/coreutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+func TestDownloadOptions_Download(t *testing.T) {
+
+ tmpFolderLocation := ""
+ tmpPrefix := "download_test"
+ inputPath := "input/inputs.pb"
+ outputPath := "output"
+
+ ctx := context.TODO()
+ dopts := DownloadOptions{
+ remoteInputsPath: inputPath,
+ remoteOutputsPrefix: outputPath,
+ metadataFormat: core.DataLoadingConfig_JSON.String(),
+ downloadMode: core.IOStrategy_DOWNLOAD_EAGER.String(),
+ }
+
+ collectFile := func(d string) []string {
+ var files []string
+ assert.NoError(t, filepath.Walk(d, func(path string, info os.FileInfo, err error) error {
+ if !strings.Contains(info.Name(), tmpPrefix) {
+ files = append(files, info.Name())
+ } // Skip tmp folder
+ return nil
+ }))
+ sort.Strings(files)
+ return files
+ }
+
+ t.Run("emptyInputs", func(t *testing.T) {
+ tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix)
+ assert.NoError(t, err)
+ defer func() {
+ assert.NoError(t, os.RemoveAll(tmpDir))
+ }()
+ dopts.localDirectoryPath = tmpDir
+
+ s := promutils.NewTestScope()
+ store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, s.NewSubScope("storage"))
+ assert.NoError(t, err)
+ dopts.RootOptions = &RootOptions{
+ Scope: s,
+ Store: store,
+ }
+
+ assert.NoError(t, store.WriteProtobuf(ctx, storage.DataReference(inputPath), storage.Options{}, &core.LiteralMap{}))
+ assert.NoError(t, dopts.Download(ctx))
+
+ assert.Len(t, collectFile(tmpDir), 0)
+ })
+
+ t.Run("primitiveInputs", func(t *testing.T) {
+ tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix)
+ assert.NoError(t, err)
+ defer func() {
+ assert.NoError(t, os.RemoveAll(tmpDir))
+ }()
+ dopts.localDirectoryPath = tmpDir
+
+ s := promutils.NewTestScope()
+ store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, s.NewSubScope("storage"))
+ assert.NoError(t, err)
+ dopts.RootOptions = &RootOptions{
+ Scope: s,
+ Store: store,
+ }
+
+ assert.NoError(t, store.WriteProtobuf(ctx, storage.DataReference(inputPath), storage.Options{}, &core.LiteralMap{
+ Literals: map[string]*core.Literal{
+ "x": coreutils.MustMakePrimitiveLiteral(1),
+ "y": coreutils.MustMakePrimitiveLiteral("hello"),
+ },
+ }))
+ assert.NoError(t, dopts.Download(ctx), "Download Operation failed")
+ assert.Equal(t, []string{"inputs.json", "inputs.pb", "x", "y"}, collectFile(tmpDir))
+ })
+
+ t.Run("primitiveAndBlobInputs", func(t *testing.T) {
+ tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix)
+ assert.NoError(t, err)
+ defer func() {
+ assert.NoError(t, os.RemoveAll(tmpDir))
+ }()
+ dopts.localDirectoryPath = tmpDir
+
+ s := promutils.NewTestScope()
+ store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, s.NewSubScope("storage"))
+ assert.NoError(t, err)
+ dopts.RootOptions = &RootOptions{
+ Scope: s,
+ Store: store,
+ }
+
+ blobLoc := storage.DataReference("blob-loc")
+ br := bytes.NewBuffer([]byte("Hello World!"))
+ assert.NoError(t, store.WriteRaw(ctx, blobLoc, int64(br.Len()), storage.Options{}, br))
+ assert.NoError(t, store.WriteProtobuf(ctx, storage.DataReference(inputPath), storage.Options{}, &core.LiteralMap{
+ Literals: map[string]*core.Literal{
+ "x": coreutils.MustMakePrimitiveLiteral(1),
+ "y": coreutils.MustMakePrimitiveLiteral("hello"),
+ "blob": {Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Blob{
+ Blob: &core.Blob{
+ Uri: blobLoc.String(),
+ Metadata: &core.BlobMetadata{
+ Type: &core.BlobType{
+ Dimensionality: core.BlobType_SINGLE,
+ Format: ".xyz",
+ },
+ },
+ },
+ },
+ },
+ }},
+ },
+ }))
+ assert.NoError(t, dopts.Download(ctx), "Download Operation failed")
+ assert.ElementsMatch(t, []string{"inputs.json", "inputs.pb", "x", "y", "blob"}, collectFile(tmpDir))
+ })
+
+ t.Run("primitiveAndMissingBlobInputs", func(t *testing.T) {
+ tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix)
+ assert.NoError(t, err)
+ defer func() {
+ assert.NoError(t, os.RemoveAll(tmpDir))
+ }()
+ dopts.localDirectoryPath = tmpDir
+
+ s := promutils.NewTestScope()
+ store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, s.NewSubScope("storage"))
+ assert.NoError(t, err)
+ dopts.RootOptions = &RootOptions{
+ Scope: s,
+ Store: store,
+ errorOutputName: "errors.pb",
+ }
+
+ assert.NoError(t, store.WriteProtobuf(ctx, storage.DataReference(inputPath), storage.Options{}, &core.LiteralMap{
+ Literals: map[string]*core.Literal{
+ "x": coreutils.MustMakePrimitiveLiteral(1),
+ "y": coreutils.MustMakePrimitiveLiteral("hello"),
+ "blob": {Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Blob{
+ Blob: &core.Blob{
+ Uri: "blob",
+ Metadata: &core.BlobMetadata{
+ Type: &core.BlobType{
+ Dimensionality: core.BlobType_SINGLE,
+ Format: ".xyz",
+ },
+ },
+ },
+ },
+ },
+ }},
+ },
+ }))
+ err = dopts.Download(ctx)
+ assert.NoError(t, err, "Download Operation failed")
+ errFile, err := store.ConstructReference(ctx, storage.DataReference(outputPath), "errors.pb")
+ assert.NoError(t, err)
+ errProto := &core.ErrorDocument{}
+ err = store.ReadProtobuf(ctx, errFile, errProto)
+ assert.NoError(t, err)
+ if assert.NotNil(t, errProto.GetError()) {
+ assert.Equal(t, core.ContainerError_RECOVERABLE, errProto.GetError().GetKind())
+ }
+ })
+}
diff --git a/flytecopilot/cmd/root.go b/flytecopilot/cmd/root.go
new file mode 100644
index 00000000000..ee6fdca010b
--- /dev/null
+++ b/flytecopilot/cmd/root.go
@@ -0,0 +1,146 @@
+package cmd
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "os"
+ "runtime"
+
+ "github.com/pkg/errors"
+ "github.com/spf13/cobra"
+ "github.com/spf13/pflag"
+ "k8s.io/client-go/tools/clientcmd"
+ "k8s.io/klog/v2"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/config"
+ "github.com/flyteorg/flyte/v2/flytestdlib/config/viper"
+ "github.com/flyteorg/flyte/v2/flytestdlib/contextutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils/labeled"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/flytestdlib/version"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+type RootOptions struct {
+ *clientcmd.ConfigOverrides
+ showSource bool
+ clientConfig clientcmd.ClientConfig
+ Scope promutils.Scope
+ Store *storage.DataStore
+ configAccessor config.Accessor
+ cfgFile string
+ // The actual key name that should be created under the remote prefix where the error document is written of the form errors.pb
+ errorOutputName string
+}
+
+func (r *RootOptions) executeRootCmd() error {
+ ctx := context.TODO()
+ logger.Infof(ctx, "Go Version: %s", runtime.Version())
+ logger.Infof(ctx, "Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH)
+ version.LogBuildInformation("flytedata")
+ return fmt.Errorf("use one of the sub-commands")
+}
+
+func (r RootOptions) UploadError(ctx context.Context, code string, recvErr error, prefix storage.DataReference) error {
+ if recvErr == nil {
+ recvErr = fmt.Errorf("unknown error")
+ }
+ errorPath, err := r.Store.ConstructReference(ctx, prefix, r.errorOutputName)
+ if err != nil {
+ logger.Errorf(ctx, "failed to create error file path err: %s", err)
+ return err
+ }
+ logger.Infof(ctx, "Uploading Error file to path [%s], errFile: %s", errorPath, r.errorOutputName)
+ return r.Store.WriteProtobuf(ctx, errorPath, storage.Options{}, &core.ErrorDocument{
+ Error: &core.ContainerError{
+ Code: code,
+ Message: recvErr.Error(),
+ Kind: core.ContainerError_RECOVERABLE,
+ },
+ })
+}
+
+// NewDataCommand returns a new instance of the co-pilot root command
+func NewDataCommand() *cobra.Command {
+ rootOpts := &RootOptions{}
+ command := &cobra.Command{
+ Use: "flytedata",
+ Short: "flytedata is a simple go binary that can be used to retrieve and upload data from/to remote stow store to local disk.",
+ Long: `flytedata when used with conjunction with flytepropeller eliminates the need to have any flyte library installed inside the container`,
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+ if err := rootOpts.initConfig(cmd, args); err != nil {
+ return err
+ }
+ rootOpts.Scope = promutils.NewScope("flyte:data")
+ cfg := storage.GetConfig()
+ store, err := storage.NewDataStore(cfg, rootOpts.Scope)
+ if err != nil {
+ return errors.Wrap(err, "failed to create datastore client")
+ }
+ rootOpts.Store = store
+ return nil
+ },
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return rootOpts.executeRootCmd()
+ },
+ }
+
+ command.AddCommand(NewDownloadCommand(rootOpts))
+ command.AddCommand(NewUploadCommand(rootOpts))
+
+ loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
+ loadingRules.DefaultClientConfig = &clientcmd.DefaultClientConfig
+ rootOpts.ConfigOverrides = &clientcmd.ConfigOverrides{}
+ kflags := clientcmd.RecommendedConfigOverrideFlags("")
+ command.PersistentFlags().StringVar(&loadingRules.ExplicitPath, "kubeconfig", "", "Path to a kube config. Only required if out-of-cluster")
+ clientcmd.BindOverrideFlags(rootOpts.ConfigOverrides, command.PersistentFlags(), kflags)
+ rootOpts.clientConfig = clientcmd.NewInteractiveDeferredLoadingClientConfig(loadingRules, rootOpts.ConfigOverrides, os.Stdin)
+
+ command.PersistentFlags().StringVar(&rootOpts.cfgFile, "config", "", "config file (default is $HOME/config.yaml)")
+ command.PersistentFlags().BoolVarP(&rootOpts.showSource, "show-source", "s", false, "Show line number for errors")
+ command.PersistentFlags().StringVar(&rootOpts.errorOutputName, "err-output-name", "errors.pb", "Actual key name under the prefix where the error protobuf should be written to")
+
+ rootOpts.configAccessor = viper.NewAccessor(config.Options{StrictMode: true})
+ // Here you will define your flags and configuration settings. Cobra supports persistent flags, which, if defined
+ // here, will be global for your application.
+ rootOpts.configAccessor.InitializePflags(command.PersistentFlags())
+
+ command.AddCommand(viper.GetConfigCommand())
+
+ return command
+}
+
+func (r *RootOptions) initConfig(cmd *cobra.Command, _ []string) error {
+ r.configAccessor = viper.NewAccessor(config.Options{
+ StrictMode: true,
+ SearchPaths: []string{r.cfgFile},
+ })
+
+ rootCmd := cmd
+ for rootCmd.Parent() != nil {
+ rootCmd = rootCmd.Parent()
+ }
+
+ // persistent flags were initially bound to the root command so we must bind to the same command to avoid
+ r.configAccessor.InitializePflags(rootCmd.PersistentFlags())
+
+ err := r.configAccessor.UpdateConfig(context.TODO())
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func init() {
+ klog.InitFlags(flag.CommandLine)
+ pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+ err := flag.CommandLine.Parse([]string{})
+ if err != nil {
+ logger.Errorf(context.TODO(), "Error in initializing: %v", err)
+ os.Exit(-1)
+ }
+ labeled.SetMetricKeys(contextutils.ProjectKey, contextutils.DomainKey, contextutils.WorkflowIDKey, contextutils.TaskIDKey)
+}
diff --git a/flytecopilot/cmd/sidecar.go b/flytecopilot/cmd/sidecar.go
new file mode 100644
index 00000000000..08baa175b19
--- /dev/null
+++ b/flytecopilot/cmd/sidecar.go
@@ -0,0 +1,155 @@
+package cmd
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/golang/protobuf/proto"
+ "github.com/pkg/errors"
+ "github.com/spf13/cobra"
+
+ "github.com/flyteorg/flyte/v2/flytecopilot/cmd/containerwatcher"
+ "github.com/flyteorg/flyte/v2/flytecopilot/data"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+const (
+ StartFile = "_START"
+ SuccessFile = "_SUCCESS"
+ ErrorFile = "_ERROR"
+)
+
+type UploadOptions struct {
+ *RootOptions
+ // The remote prefix where all the meta outputs or error should be uploaded of the form s3://bucket/prefix
+ remoteOutputsPrefix string
+ // Name like outputs.pb under the remoteOutputsPrefix that should be created to upload the metaOutputs
+ metaOutputName string
+ // The remote prefix where all the raw outputs should be uploaded of the form s3://bucket/prefix/
+ remoteOutputsRawPrefix string
+ // Local directory path where the sidecar should look for outputs.
+ localDirectoryPath string
+ // Non primitive types will be dumped in this output format
+ metadataFormat string
+ uploadMode string
+ timeout time.Duration
+ typedInterface []byte
+ startWatcherType containerwatcher.WatcherType
+ exitWatcherType containerwatcher.WatcherType
+}
+
+func (u *UploadOptions) createWatcher(_ context.Context, w containerwatcher.WatcherType) (containerwatcher.Watcher, error) {
+ switch w {
+ case containerwatcher.WatcherTypeSignal:
+ return containerwatcher.SignalWatcher{}, nil
+ case containerwatcher.WatcherTypeNoop:
+ return containerwatcher.NoopWatcher{}, nil
+ }
+ return nil, fmt.Errorf("unsupported watcher type")
+}
+
+func (u *UploadOptions) uploader(ctx context.Context) error {
+ if u.typedInterface == nil {
+ logger.Infof(ctx, "No output interface provided. Assuming Void outputs.")
+ return nil
+ }
+
+ iface := &core.TypedInterface{}
+ if err := proto.Unmarshal(u.typedInterface, iface); err != nil {
+ logger.Errorf(ctx, "Bad interface passed, failed to unmarshal err: %s", err)
+ return errors.Wrap(err, "Bad interface passed, failed to unmarshal, expected core.TypedInterface")
+ }
+ outputInterface := iface.GetOutputs()
+
+ if iface.GetOutputs() == nil || iface.Outputs.Variables == nil || len(iface.GetOutputs().GetVariables()) == 0 {
+ logger.Infof(ctx, "Empty output interface received. Assuming void outputs. Sidecar will exit immediately.")
+ return nil
+ }
+
+ f, ok := core.DataLoadingConfig_LiteralMapFormat_value[u.metadataFormat]
+ if !ok {
+ return fmt.Errorf("incorrect input data format specified, given [%s], possible values [%+v]", u.metadataFormat, GetFormatVals())
+ }
+
+ m, ok := core.IOStrategy_UploadMode_value[u.uploadMode]
+ if !ok {
+ return fmt.Errorf("incorrect input upload mode specified, given [%s], possible values [%+v]", u.uploadMode, GetUploadModeVals())
+ }
+
+ logger.Infof(ctx, "Creating start watcher type: %s", u.startWatcherType)
+ w, err := u.createWatcher(ctx, u.startWatcherType)
+ if err != nil {
+ return err
+ }
+
+ logger.Infof(ctx, "Waiting for Container to exit.")
+ if err := w.WaitToExit(ctx); err != nil {
+ logger.Errorf(ctx, "Failed waiting for container to exit. Err: %s", err)
+ return err
+ }
+
+ logger.Infof(ctx, "Container Exited! uploading data.")
+
+ // TODO maybe we should just take the meta output path as an input argument
+ toOutputPath, err := u.Store.ConstructReference(ctx, storage.DataReference(u.remoteOutputsPrefix), u.metaOutputName)
+ if err != nil {
+ return err
+ }
+
+ dl := data.NewUploader(ctx, u.Store, core.DataLoadingConfig_LiteralMapFormat(f), core.IOStrategy_UploadMode(m), ErrorFile)
+
+ childCtx, cancelFn := context.WithTimeout(ctx, u.timeout)
+ defer cancelFn()
+ if err := dl.RecursiveUpload(childCtx, outputInterface, u.localDirectoryPath, toOutputPath, storage.DataReference(u.remoteOutputsRawPrefix)); err != nil {
+ logger.Errorf(ctx, "Uploading failed, err %s", err)
+ return err
+ }
+
+ logger.Infof(ctx, "Uploader completed successfully!")
+ return nil
+}
+
+func (u *UploadOptions) Sidecar(ctx context.Context) error {
+
+ if err := u.uploader(ctx); err != nil {
+ logger.Errorf(ctx, "Uploading failed, err %s", err)
+ if err := u.UploadError(ctx, "OutputUploadFailed", err, storage.DataReference(u.remoteOutputsPrefix)); err != nil {
+ logger.Errorf(ctx, "Failed to write error document, err :%s", err)
+ return err
+ }
+ }
+ return nil
+}
+
+func NewUploadCommand(opts *RootOptions) *cobra.Command {
+
+ uploadOptions := &UploadOptions{
+ RootOptions: opts,
+ }
+
+ // deleteCmd represents the delete command
+ uploadCmd := &cobra.Command{
+ Use: "sidecar ",
+ Short: "uploads flyteData from the localpath to a remote dir.",
+ Long: `Currently it looks at the outputs.pb and creates one file per variable.`,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return uploadOptions.Sidecar(context.Background())
+ },
+ }
+
+ uploadCmd.Flags().StringVarP(&uploadOptions.remoteOutputsPrefix, "to-output-prefix", "o", "", "The remote path/key prefix for output metadata in stow store.")
+ uploadCmd.Flags().StringVarP(&uploadOptions.remoteOutputsRawPrefix, "to-raw-output", "x", "", "The remote path/key prefix for outputs in remote store. This is a sandbox directory and all data will be uploaded here.")
+ uploadCmd.Flags().StringVarP(&uploadOptions.localDirectoryPath, "from-local-dir", "f", "", "The local directory on disk where data will be available for upload.")
+ uploadCmd.Flags().StringVarP(&uploadOptions.metadataFormat, "format", "m", core.DataLoadingConfig_JSON.String(), fmt.Sprintf("What should be the output format for the primitive and structured types. Options [%v]", GetFormatVals()))
+ uploadCmd.Flags().StringVarP(&uploadOptions.uploadMode, "upload-mode", "u", core.IOStrategy_UPLOAD_ON_EXIT.String(), fmt.Sprintf("When should upload start/upload mode. Options [%v]", GetUploadModeVals()))
+ uploadCmd.Flags().StringVarP(&uploadOptions.metaOutputName, "meta-output-name", "", "outputs.pb", "The key name under the remoteOutputPrefix that should be return to provide meta information about the outputs on successful execution")
+ uploadCmd.Flags().DurationVarP(&uploadOptions.timeout, "timeout", "t", time.Hour*1, "Max time to allow for uploads to complete, default is 1H")
+ uploadCmd.Flags().BytesBase64VarP(&uploadOptions.typedInterface, "interface", "i", nil, "Typed Interface - core.TypedInterface, base64 encoded string of the serialized protobuf")
+ uploadCmd.Flags().DurationVarP(&uploadOptions.timeout, "start-timeout", "", 0, "Deprecated: Use --timeout instead. Specifies the maximum duration to allow uploads to complete. Retained for backward compatibility.")
+ uploadCmd.Flags().StringVarP(&uploadOptions.startWatcherType, "start-watcher-type", "", containerwatcher.WatcherTypeSignal, fmt.Sprintf("Sidecar will wait for container before starting upload process. Watcher type makes the type configurable. Available Type %+v", containerwatcher.AllWatcherTypes))
+ uploadCmd.Flags().StringVarP(&uploadOptions.exitWatcherType, "exit-watcher-type", "", containerwatcher.WatcherTypeSignal, fmt.Sprintf("Sidecar will wait for completion of the container before starting upload process. Watcher type makes the type configurable. Available Type %+v", containerwatcher.AllWatcherTypes))
+ return uploadCmd
+}
diff --git a/flytecopilot/cmd/sidecar_test.go b/flytecopilot/cmd/sidecar_test.go
new file mode 100644
index 00000000000..c8a89c60e55
--- /dev/null
+++ b/flytecopilot/cmd/sidecar_test.go
@@ -0,0 +1,96 @@
+package cmd
+
+import (
+ "context"
+ "io/ioutil"
+ "os"
+ "testing"
+
+ "github.com/golang/protobuf/proto"
+ "github.com/stretchr/testify/assert"
+
+ "github.com/flyteorg/flyte/v2/flytecopilot/cmd/containerwatcher"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+func TestUploadOptions_Upload(t *testing.T) {
+ tmpFolderLocation := ""
+ tmpPrefix := "upload_test"
+ outputPath := "output"
+
+ ctx := context.TODO()
+
+ t.Run("uploadNoOutputs", func(t *testing.T) {
+ tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix)
+ assert.NoError(t, err)
+ defer func() {
+ assert.NoError(t, os.RemoveAll(tmpDir))
+ }()
+
+ s := promutils.NewTestScope()
+ store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, s.NewSubScope("storage"))
+ assert.NoError(t, err)
+ uopts := UploadOptions{
+ RootOptions: &RootOptions{
+ Scope: s,
+ Store: store,
+ },
+ remoteOutputsPrefix: outputPath,
+ metadataFormat: core.DataLoadingConfig_JSON.String(),
+ uploadMode: core.IOStrategy_UPLOAD_ON_EXIT.String(),
+ startWatcherType: containerwatcher.WatcherTypeNoop,
+ localDirectoryPath: tmpDir,
+ }
+
+ assert.NoError(t, uopts.Sidecar(ctx))
+ })
+
+ t.Run("uploadBlobType-FileNotFound", func(t *testing.T) {
+ tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix)
+ assert.NoError(t, err)
+ defer func() {
+ assert.NoError(t, os.RemoveAll(tmpDir))
+ }()
+ s := promutils.NewTestScope()
+ store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, s.NewSubScope("storage"))
+ assert.NoError(t, err)
+
+ iface := &core.TypedInterface{
+ Outputs: &core.VariableMap{
+ Variables: []*core.VariableEntry{
+ {
+ Key: "x",
+ Value: &core.Variable{
+ Type: &core.LiteralType{Type: &core.LiteralType_Blob{Blob: &core.BlobType{Dimensionality: core.BlobType_SINGLE}}},
+ Description: "example",
+ },
+ },
+ },
+ },
+ }
+ d, err := proto.Marshal(iface)
+ assert.NoError(t, err)
+
+ uopts := UploadOptions{
+ RootOptions: &RootOptions{
+ Scope: s,
+ Store: store,
+ errorOutputName: "errors.pb",
+ },
+ remoteOutputsPrefix: outputPath,
+ metadataFormat: core.DataLoadingConfig_JSON.String(),
+ uploadMode: core.IOStrategy_UPLOAD_ON_EXIT.String(),
+ startWatcherType: containerwatcher.WatcherTypeNoop,
+ exitWatcherType: containerwatcher.WatcherTypeNoop,
+ typedInterface: d,
+ localDirectoryPath: tmpDir,
+ }
+
+ assert.NoError(t, uopts.Sidecar(ctx))
+ v, err := store.Head(ctx, "/output/errors.pb")
+ assert.NoError(t, err)
+ assert.True(t, v.Exists())
+ })
+}
diff --git a/flytecopilot/data/common.go b/flytecopilot/data/common.go
new file mode 100644
index 00000000000..132d57b37b0
--- /dev/null
+++ b/flytecopilot/data/common.go
@@ -0,0 +1,22 @@
+// This module contains Flyte CoPilot related code.
+// Currently it only has 2 utilities - downloader and an uploader.
+// Usage Downloader:
+//
+// downloader := NewDownloader(...)
+// downloader.DownloadInputs(...) // will recursively download all inputs
+//
+// Usage uploader:
+//
+// uploader := NewUploader(...)
+// uploader.RecursiveUpload(...) // Will recursively upload all the data from the given path depending on the output interface
+//
+// All errors are bubbled up.
+//
+// Both the uploader and downloader accept context.Context variables. These should be used to control timeouts etc.
+// TODO: Currently retries are not automatically handled.
+package data
+
+import "github.com/flyteorg/flyte/v2/flytestdlib/futures"
+
+type VarMap map[string]interface{}
+type FutureMap map[string]futures.Future
diff --git a/flytecopilot/data/download.go b/flytecopilot/data/download.go
new file mode 100644
index 00000000000..853487cfa28
--- /dev/null
+++ b/flytecopilot/data/download.go
@@ -0,0 +1,575 @@
+package data
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "os"
+ "path"
+ "path/filepath"
+ "reflect"
+ "strconv"
+ "strings"
+ "sync"
+
+ "github.com/ghodss/yaml"
+ "github.com/golang/protobuf/jsonpb"
+ "github.com/golang/protobuf/proto"
+ "github.com/golang/protobuf/ptypes"
+ structpb "github.com/golang/protobuf/ptypes/struct"
+ "github.com/pkg/errors"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/futures"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+type Downloader struct {
+ format core.DataLoadingConfig_LiteralMapFormat
+ store *storage.DataStore
+ // TODO support download mode
+ mode core.IOStrategy_DownloadMode
+}
+
+// TODO add timeout and rate limit
+// TODO use chunk to download
+func (d Downloader) handleBlob(ctx context.Context, blob *core.Blob, toPath string) (interface{}, error) {
+ /*
+ handleBlob handles the retrieval and local storage of blob data, including support for both single and multipart blob types.
+ For multipart blobs, it lists all parts recursively and spawns concurrent goroutines to download each part while managing file I/O in parallel.
+
+ - The function begins by validating the blob URI and categorizing the blob type (single or multipart).
+ - In the multipart case, it recursively lists all blob parts and launches goroutines to download and save each part.
+ Goroutine closure and I/O success tracking are managed to avoid resource leaks.
+ - For single-part blobs, it directly downloads and writes the data to the specified path.
+
+ Life Cycle:
+ 1. Blob URI -> Blob Metadata Type check -> Recursive List parts if Multipart -> Launch goroutines to download parts
+ (input blob object) (determine multipart/single) (List API, handles recursive case) (each part handled in parallel)
+ 2. Download part or full blob -> Save locally with error checks -> Handle reader/writer closures -> Return local path or error
+ (download each part) (error on write or directory) (close streams safely, track success) (completion or report missing closures)
+
+ More clarification on Folders. If a user returns FlyteFile("/my/folder") and inside /my/folder is
+ - sample.txt
+ - nested/
+ - deep_file.txt
+
+ The blob uri is something like: s3://container/oz/a9ss8w4mnkk8zttr9p7z-n0-0/0fda6abb7cd4e45cfae4920f0b8a586d
+ and inside are the files:
+ - s3://container/oz/a9ss8w4mnkk8zttr9p7z-n0-0/0fda6abb7cd4e45cfae4920f0b8a586d/sample.txt
+ - s3://container/oz/a9ss8w4mnkk8zttr9p7z-n0-0/0fda6abb7cd4e45cfae4920f0b8a586d/nested/deep_file.txt
+ the top level folder disappears. If we want t
+ */
+
+ blobRef := storage.DataReference(blob.GetUri())
+ scheme, baseContainer, basePrefix, err := blobRef.Split()
+ logger.Debugf(ctx, "Downloader handling blob [%s] uri [%s] in bucket [%s] prefix [%s]", scheme, blob.GetUri(), baseContainer, basePrefix)
+ if err != nil {
+ return nil, errors.Wrapf(err, "Blob uri incorrectly formatted")
+ }
+
+ if blob.GetMetadata().GetType().GetDimensionality() == core.BlobType_MULTIPART {
+ // Collect all parts of the multipart blob recursively (List API handles nested directories)
+ // Set maxItems to 100 as a parameter for the List API, enabling batch retrieval of items until all are downloaded
+ maxItems := 100
+ cursor := storage.NewCursorAtStart()
+ var items []storage.DataReference
+ var absPaths []string
+
+ for {
+ items, cursor, err = d.store.List(ctx, blobRef, maxItems, cursor)
+ if err != nil || len(items) == 0 {
+ logger.Errorf(ctx, "failed to collect items from multipart blob [%s]", blobRef)
+ return nil, err
+ }
+ for _, item := range items {
+ absPaths = append(absPaths, item.String())
+ }
+ if storage.IsCursorEnd(cursor) {
+ break
+ }
+ }
+
+ // Track the count of successful downloads and the total number of items
+ downloadSuccess := 0
+ itemCount := len(absPaths)
+ // Track successful closures of readers and writers in deferred functions
+ readerCloseSuccessCount := 0
+ writerCloseSuccessCount := 0
+ // We use Mutex to avoid race conditions when updating counters and creating directories
+ var mu sync.Mutex
+ var wg sync.WaitGroup
+ for _, absPath := range absPaths {
+ absPath := absPath
+
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ defer func() {
+ if err := recover(); err != nil {
+ logger.Errorf(ctx, "recover receives error: [%s]", err)
+ }
+ }()
+
+ ref := storage.DataReference(absPath)
+ scheme, _, prefix, err := ref.Split()
+ if err != nil {
+ logger.Errorf(ctx, "Failed to parse [%s] [%s]", ref, err)
+ return
+ }
+ var reader io.ReadCloser
+ if scheme == "http" || scheme == "https" {
+ reader, err = DownloadFileFromHTTP(ctx, ref)
+ } else {
+ reader, err = DownloadFileFromStorage(ctx, ref, d.store)
+ }
+ if err != nil {
+ logger.Errorf(ctx, "Failed to download from ref [%s]", ref)
+ return
+ }
+ defer func() {
+ err := reader.Close()
+ if err != nil {
+ logger.Errorf(ctx, "failed to close Blob read stream @ref [%s].\n"+
+ "Error: %s", ref, err)
+ }
+ mu.Lock()
+ readerCloseSuccessCount++
+ mu.Unlock()
+ }()
+
+ // Strip the base path from the item prefix to get the relative path
+ // For HTTP/HTTPS URLs: prefix includes bucket + path (e.g., "bucket/sm/akm6s4bgd6lwx6fhzf58-n0-0/705fe4570586b256a5b0e5fd598b4c28/sample.txt")
+ // For S3/GS URLs: prefix only includes path (e.g., "sm/akm6s4bgd6lwx6fhzf58-n0-0/705fe4570586b256a5b0e5fd598b4c28/sample.txt")
+ // We need to strip the blob's base path to get just the relative file path
+ relativePath := prefix
+
+ // Strip the base path from the item prefix to get the relative path
+ // This works for both HTTP and native cloud storage URLs:
+ // - HTTP: prefix="bucket/path/file.txt", basePrefix="path"
+ // - S3/GS: prefix="path/file.txt", basePrefix="path"
+ if strings.HasPrefix(absPath, "http") {
+ // Try matching two ways...
+ logger.Debugf(ctx, "matching with container, prefix=[%s] %s", prefix, baseContainer+"/"+basePrefix)
+ if strings.HasPrefix(prefix, baseContainer) {
+ // This works for S3
+ relativePath = strings.TrimPrefix(prefix, baseContainer+"/"+basePrefix)
+ } else {
+ // This is here because google has the aggravating behavior of injecting a /o/ into the download
+ // link so we can't use the above. instead just look for the basePrefix in the prefix.
+ // This should work for S3 too, but using the base container feels safer.
+ idx := strings.Index(prefix, basePrefix)
+ if idx == -1 {
+ logger.Errorf(ctx, "Failed to find container prefix [%s]", prefix)
+ } else {
+ // Extract everything after basePrefix
+ relativePath = prefix[idx+len(basePrefix):]
+ }
+ }
+ } else {
+ logger.Debugf(ctx, "matching prefix=[%s] %s", prefix, basePrefix)
+ relativePath = strings.TrimPrefix(prefix, basePrefix)
+ }
+ // Remove leading slash if it exists
+ relativePath = strings.TrimPrefix(relativePath, "/")
+ logger.Debugf(ctx, "Extracting file from %s, using relative path %s", absPath, relativePath)
+
+ newPath := filepath.Join(toPath, relativePath)
+ dir := filepath.Dir(newPath)
+
+ mu.Lock()
+ // os.MkdirAll creates the specified directory structure if it doesn’t already exist
+ // 0777: the directory can be read and written by anyone
+ err = os.MkdirAll(dir, 0777)
+ mu.Unlock()
+ if err != nil {
+ logger.Errorf(ctx, "failed to make dir at path [%s]", dir)
+ return
+ }
+
+ writer, err := os.Create(newPath)
+ if err != nil {
+ logger.Errorf(ctx, "failed to open file at path [%s]", newPath)
+ return
+ }
+ defer func() {
+ err := writer.Close()
+ if err != nil {
+ logger.Errorf(ctx, "failed to close File write stream.\n"+
+ "Error: [%s]", err)
+ }
+ mu.Lock()
+ writerCloseSuccessCount++
+ mu.Unlock()
+ }()
+
+ _, err = io.Copy(writer, reader)
+ if err != nil {
+ logger.Errorf(ctx, "failed to write remote data to local filesystem")
+ return
+ }
+ mu.Lock()
+ downloadSuccess++
+ mu.Unlock()
+ }()
+ }
+ // Go routines are synchronized with a WaitGroup to prevent goroutine leaks.
+ wg.Wait()
+ if downloadSuccess != itemCount || readerCloseSuccessCount != itemCount || writerCloseSuccessCount != itemCount {
+ return nil, errors.Errorf(
+ "Failed to copy %d out of %d remote files from [%s] to local [%s].\n"+
+ "Failed to close %d readers\n"+
+ "Failed to close %d writers.",
+ itemCount-downloadSuccess, itemCount, blobRef, toPath, itemCount-readerCloseSuccessCount, itemCount-writerCloseSuccessCount,
+ )
+ }
+ logger.Infof(ctx, "successfully copied %d remote files from [%s] to local [%s]", downloadSuccess, blobRef, toPath)
+ return toPath, nil
+ } else if blob.GetMetadata().GetType().GetDimensionality() == core.BlobType_SINGLE {
+ // reader should be declared here (avoid being shared across all goroutines)
+ var reader io.ReadCloser
+ if scheme == "http" || scheme == "https" {
+ reader, err = DownloadFileFromHTTP(ctx, blobRef)
+ } else {
+ reader, err = DownloadFileFromStorage(ctx, blobRef, d.store)
+ }
+ if err != nil {
+ logger.Errorf(ctx, "Failed to download from ref [%s]", blobRef)
+ return nil, err
+ }
+ defer func() {
+ err := reader.Close()
+ if err != nil {
+ logger.Errorf(ctx, "failed to close Blob read stream @ref [%s]. Error: %s", blobRef, err)
+ }
+ }()
+
+ writer, err := os.Create(toPath)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to open file at path %s", toPath)
+ }
+ defer func() {
+ err := writer.Close()
+ if err != nil {
+ logger.Errorf(ctx, "failed to close File write stream. Error: %s", err)
+ }
+ }()
+ v, err := io.Copy(writer, reader)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to write remote data to local filesystem")
+ }
+ logger.Infof(ctx, "Successfully copied [%d] bytes remote data from [%s] to local [%s]", v, blobRef, toPath)
+ return toPath, nil
+ }
+
+ return nil, errors.Errorf("unexpected Blob type encountered")
+}
+
+func (d Downloader) handleSchema(ctx context.Context, schema *core.Schema, toFilePath string) (interface{}, error) {
+ return d.handleBlob(ctx, &core.Blob{Uri: schema.GetUri(), Metadata: &core.BlobMetadata{Type: &core.BlobType{Dimensionality: core.BlobType_MULTIPART}}}, toFilePath)
+}
+
+func (d Downloader) handleBinary(_ context.Context, b *core.Binary, toFilePath string, writeToFile bool) (interface{}, error) {
+ // maybe we should return a map
+ v := b.GetValue()
+ if writeToFile {
+ return v, os.WriteFile(toFilePath, v, os.ModePerm) // #nosec G306
+ }
+ return v, nil
+}
+
+func (d Downloader) handleError(_ context.Context, b *core.Error, toFilePath string, writeToFile bool) (interface{}, error) {
+ // maybe we should return a map
+ if writeToFile {
+ return b.GetMessage(), os.WriteFile(toFilePath, []byte(b.GetMessage()), os.ModePerm) // #nosec G306
+ }
+ return b.GetMessage(), nil
+}
+
+func (d Downloader) handleGeneric(ctx context.Context, b *structpb.Struct, toFilePath string, writeToFile bool) (interface{}, error) {
+ if writeToFile && b != nil {
+ m := jsonpb.Marshaler{}
+ writer, err := os.Create(toFilePath)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to open file at path %s", toFilePath)
+ }
+ defer func() {
+ err := writer.Close()
+ if err != nil {
+ logger.Errorf(ctx, "failed to close File write stream. Error: %s", err)
+ }
+ }()
+ return b, m.Marshal(writer, b)
+ }
+ return b, nil
+}
+
+// Returns the primitive value in Golang native format and if the filePath is not empty, then writes the value to the given file path.
+func (d Downloader) handlePrimitive(primitive *core.Primitive, toFilePath string, writeToFile bool) (interface{}, error) {
+
+ var toByteArray func() ([]byte, error)
+ var v interface{}
+ var err error
+
+ switch primitive.GetValue().(type) {
+ case *core.Primitive_StringValue:
+ v = primitive.GetStringValue()
+ toByteArray = func() ([]byte, error) {
+ return []byte(primitive.GetStringValue()), nil
+ }
+ case *core.Primitive_Boolean:
+ v = primitive.GetBoolean()
+ toByteArray = func() ([]byte, error) {
+ return []byte(strconv.FormatBool(primitive.GetBoolean())), nil
+ }
+ case *core.Primitive_Integer:
+ v = primitive.GetInteger()
+ toByteArray = func() ([]byte, error) {
+ return []byte(strconv.FormatInt(primitive.GetInteger(), 10)), nil
+ }
+ case *core.Primitive_FloatValue:
+ v = primitive.GetFloatValue()
+ toByteArray = func() ([]byte, error) {
+ return []byte(strconv.FormatFloat(primitive.GetFloatValue(), 'f', -1, 64)), nil
+ }
+ case *core.Primitive_Datetime:
+ v, err = ptypes.Timestamp(primitive.GetDatetime())
+ if err != nil {
+ return nil, err
+ }
+ toByteArray = func() ([]byte, error) {
+ return []byte(ptypes.TimestampString(primitive.GetDatetime())), nil
+ }
+ case *core.Primitive_Duration:
+ v, err := ptypes.Duration(primitive.GetDuration())
+ if err != nil {
+ return nil, err
+ }
+ toByteArray = func() ([]byte, error) {
+ return []byte(v.String()), nil
+ }
+ default:
+ v = nil
+ toByteArray = func() ([]byte, error) {
+ return []byte("null"), nil
+ }
+ }
+ if writeToFile {
+ b, err := toByteArray()
+ if err != nil {
+ return nil, err
+ }
+ return v, os.WriteFile(toFilePath, b, os.ModePerm) // #nosec G306
+ }
+ return v, nil
+}
+
+func (d Downloader) handleScalar(ctx context.Context, scalar *core.Scalar, toFilePath string, writeToFile bool) (interface{}, *core.Scalar, error) {
+ switch scalar.GetValue().(type) {
+ case *core.Scalar_Primitive:
+ p := scalar.GetPrimitive()
+ i, err := d.handlePrimitive(p, toFilePath, writeToFile)
+ return i, scalar, err
+ case *core.Scalar_Blob:
+ b := scalar.GetBlob()
+ i, err := d.handleBlob(ctx, b, toFilePath)
+ return i, &core.Scalar{Value: &core.Scalar_Blob{Blob: &core.Blob{Metadata: b.GetMetadata(), Uri: toFilePath}}}, err
+ case *core.Scalar_Schema:
+ b := scalar.GetSchema()
+ i, err := d.handleSchema(ctx, b, toFilePath)
+ return i, &core.Scalar{Value: &core.Scalar_Schema{Schema: &core.Schema{Type: b.GetType(), Uri: toFilePath}}}, err
+ case *core.Scalar_Binary:
+ b := scalar.GetBinary()
+ i, err := d.handleBinary(ctx, b, toFilePath, writeToFile)
+ return i, scalar, err
+ case *core.Scalar_Error:
+ b := scalar.GetError()
+ i, err := d.handleError(ctx, b, toFilePath, writeToFile)
+ return i, scalar, err
+ case *core.Scalar_Generic:
+ b := scalar.GetGeneric()
+ i, err := d.handleGeneric(ctx, b, toFilePath, writeToFile)
+ return i, scalar, err
+ case *core.Scalar_Union:
+ b := scalar.GetUnion()
+ i, lit, err := d.handleLiteral(ctx, b.GetValue(), toFilePath, writeToFile)
+ return i, &core.Scalar{Value: &core.Scalar_Union{Union: &core.Union{Type: b.GetType(), Value: lit}}}, err
+ case *core.Scalar_NoneType:
+ if writeToFile {
+ return nil, scalar, os.WriteFile(toFilePath, []byte("null"), os.ModePerm) // #nosec G306
+ }
+ return nil, scalar, nil
+ default:
+ return nil, nil, fmt.Errorf("unsupported scalar type [%v]", reflect.TypeOf(scalar.GetValue()))
+ }
+}
+
+func (d Downloader) handleLiteral(ctx context.Context, lit *core.Literal, filePath string, writeToFile bool) (interface{}, *core.Literal, error) {
+ switch lit.GetValue().(type) {
+ case *core.Literal_Scalar:
+ v, s, err := d.handleScalar(ctx, lit.GetScalar(), filePath, writeToFile)
+ if err != nil {
+ return nil, nil, err
+ }
+ return v, &core.Literal{Value: &core.Literal_Scalar{
+ Scalar: s,
+ }}, nil
+ case *core.Literal_Collection:
+ err := os.MkdirAll(filePath, os.ModePerm)
+ if err != nil {
+ return nil, nil, errors.Wrapf(err, "failed to create directory [%s]", filePath)
+ }
+ v, c2, err := d.handleCollection(ctx, lit.GetCollection(), filePath, writeToFile)
+ if err != nil {
+ return nil, nil, err
+ }
+ return v, &core.Literal{Value: &core.Literal_Collection{
+ Collection: c2,
+ }}, nil
+ case *core.Literal_Map:
+ err := os.MkdirAll(filePath, os.ModePerm)
+ if err != nil {
+ return nil, nil, errors.Wrapf(err, "failed to create directory [%s]", filePath)
+ }
+ v, m, err := d.RecursiveDownload(ctx, lit.GetMap(), filePath, writeToFile)
+ if err != nil {
+ return nil, nil, err
+ }
+ return v, &core.Literal{Value: &core.Literal_Map{
+ Map: m,
+ }}, nil
+ default:
+ return nil, nil, fmt.Errorf("unsupported literal type [%v]", reflect.TypeOf(lit.GetValue()))
+ }
+}
+
+// Collection should be stored as a top level list file and may have accompanying files?
+func (d Downloader) handleCollection(ctx context.Context, c *core.LiteralCollection, dir string, writePrimitiveToFile bool) ([]interface{}, *core.LiteralCollection, error) {
+ if c == nil || len(c.GetLiterals()) == 0 {
+ return []interface{}{}, c, nil
+ }
+ var collection []interface{}
+ litCollection := &core.LiteralCollection{}
+ for i, lit := range c.GetLiterals() {
+ filePath := path.Join(dir, strconv.Itoa(i))
+ v, lit, err := d.handleLiteral(ctx, lit, filePath, writePrimitiveToFile)
+ if err != nil {
+ return nil, nil, err
+ }
+ collection = append(collection, v)
+ litCollection.Literals = append(litCollection.Literals, lit)
+ }
+ return collection, litCollection, nil
+}
+
+type downloadedResult struct {
+ lit *core.Literal
+ v interface{}
+}
+
+func (d Downloader) RecursiveDownload(ctx context.Context, inputs *core.LiteralMap, dir string, writePrimitiveToFile bool) (VarMap, *core.LiteralMap, error) {
+ childCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ if inputs == nil || len(inputs.GetLiterals()) == 0 {
+ return VarMap{}, nil, nil
+ }
+ f := make(FutureMap, len(inputs.GetLiterals()))
+ for variable, literal := range inputs.GetLiterals() {
+ if literal.GetOffloadedMetadata() != nil {
+ offloadedMetadataURI := literal.GetOffloadedMetadata().GetUri()
+ // literal will be overwritten with the contents of the offloaded data which contains the actual large literal.
+ if err := d.store.ReadProtobuf(ctx, storage.DataReference(offloadedMetadataURI), literal); err != nil {
+ errString := fmt.Sprintf("Failed to read the object at location [%s] with error [%s]", offloadedMetadataURI, err)
+ logger.Error(ctx, errString)
+ return nil, nil, fmt.Errorf("%s", errString)
+ }
+ logger.Infof(ctx, "read object at location [%s]", offloadedMetadataURI)
+ }
+ varPath := path.Join(dir, variable)
+ lit := literal
+ f[variable] = futures.NewAsyncFuture(childCtx, func(ctx2 context.Context) (interface{}, error) {
+ v, lit, err := d.handleLiteral(ctx2, lit, varPath, writePrimitiveToFile)
+ if err != nil {
+ return nil, err
+ }
+ return downloadedResult{lit: lit, v: v}, nil
+ })
+ }
+
+ m := &core.LiteralMap{
+ Literals: make(map[string]*core.Literal),
+ }
+ vmap := make(VarMap, len(f))
+ for variable, future := range f {
+ logger.Infof(ctx, "Waiting for [%s] to be persisted", variable)
+ v, err := future.Get(childCtx)
+ if err != nil {
+ logger.Errorf(ctx, "Failed to persist [%s], err %s", variable, err)
+ if err == futures.ErrAsyncFutureCanceled {
+ logger.Errorf(ctx, "Future was canceled, possibly Timeout!")
+ }
+ return nil, nil, errors.Wrapf(err, "variable [%s] download/store failed", variable)
+ }
+ dr := v.(downloadedResult)
+ vmap[variable] = dr.v
+ m.Literals[variable] = dr.lit
+ logger.Infof(ctx, "Completed persisting [%s]", variable)
+ }
+
+ return vmap, m, nil
+}
+
+func (d Downloader) DownloadInputs(ctx context.Context, inputRef storage.DataReference, outputDir string) error {
+ logger.Infof(ctx, "Downloading inputs from [%s]", inputRef)
+ defer logger.Infof(ctx, "Exited downloading inputs from [%s]", inputRef)
+ if err := os.MkdirAll(outputDir, os.ModePerm); err != nil {
+ logger.Errorf(ctx, "Failed to create output directories, err: %s", err)
+ return err
+ }
+ inputs := &core.LiteralMap{}
+ err := d.store.ReadProtobuf(ctx, inputRef, inputs)
+ if err != nil {
+ logger.Errorf(ctx, "Failed to download inputs from [%s], err [%s]", inputRef, err)
+ return errors.Wrapf(err, "failed to download input metadata message from remote store")
+ }
+ varMap, lMap, err := d.RecursiveDownload(ctx, inputs, outputDir, true)
+ if err != nil {
+ return errors.Wrapf(err, "failed to download input variable from remote store")
+ }
+
+ // We will always write the protobuf
+ b, err := proto.Marshal(lMap)
+ if err != nil {
+ return err
+ }
+ // #nosec G306
+ if err := os.WriteFile(path.Join(outputDir, "inputs.pb"), b, os.ModePerm); err != nil {
+ return err
+ }
+
+ if d.format == core.DataLoadingConfig_JSON {
+ m, err := json.Marshal(varMap)
+ if err != nil {
+ return errors.Wrapf(err, "failed to marshal out inputs")
+ }
+ return os.WriteFile(path.Join(outputDir, "inputs.json"), m, os.ModePerm) // #nosec G306
+ }
+ if d.format == core.DataLoadingConfig_YAML {
+ m, err := yaml.Marshal(varMap)
+ if err != nil {
+ return errors.Wrapf(err, "failed to marshal out inputs")
+ }
+ return os.WriteFile(path.Join(outputDir, "inputs.yaml"), m, os.ModePerm) // #nosec G306
+ }
+ return nil
+}
+
+func NewDownloader(_ context.Context, store *storage.DataStore, format core.DataLoadingConfig_LiteralMapFormat, mode core.IOStrategy_DownloadMode) Downloader {
+ return Downloader{
+ format: format,
+ store: store,
+ mode: mode,
+ }
+}
diff --git a/flytecopilot/data/download_test.go b/flytecopilot/data/download_test.go
new file mode 100644
index 00000000000..2fb23847f63
--- /dev/null
+++ b/flytecopilot/data/download_test.go
@@ -0,0 +1,413 @@
+package data
+
+import (
+ "bytes"
+ "context"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+func TestHandleBlobMultipart(t *testing.T) {
+ t.Run("Successful Query", func(t *testing.T) {
+ s, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, promutils.NewTestScope())
+ assert.NoError(t, err)
+
+ // Create one file at root level and another in a nested folder
+ ref1 := storage.DataReference("s3://container/oz/a9ss8w4mnkk8zttr9p7z-n0-0/0fda6abb7c/root_file.txt")
+ err = s.WriteRaw(context.Background(), ref1, 0, storage.Options{}, bytes.NewReader([]byte("root content")))
+ assert.NoError(t, err)
+
+ ref2 := storage.DataReference("s3://container/oz/a9ss8w4mnkk8zttr9p7z-n0-0/0fda6abb7c/nested/deep_file.txt")
+ err = s.WriteRaw(context.Background(), ref2, 0, storage.Options{}, bytes.NewReader([]byte("nested content")))
+ assert.NoError(t, err)
+
+ d := Downloader{store: s}
+
+ blob := &core.Blob{
+ Uri: "s3://container/oz/a9ss8w4mnkk8zttr9p7z-n0-0/0fda6abb7c",
+ Metadata: &core.BlobMetadata{
+ Type: &core.BlobType{
+ Dimensionality: core.BlobType_MULTIPART,
+ },
+ },
+ }
+
+ // Create temporary directory with inputs/my_var structure
+ tmpDir, err := os.MkdirTemp("", "blob_test")
+ assert.NoError(t, err)
+ testPath := filepath.Join(tmpDir, "inputs", "my_var")
+ defer func() {
+ err := os.RemoveAll(tmpDir)
+ if err != nil {
+ t.Errorf("Failed to delete directory: %v", err)
+ }
+ }()
+
+ result, err := d.handleBlob(context.Background(), blob, testPath)
+ assert.NoError(t, err)
+ assert.Equal(t, testPath, result)
+
+ // Check if files were created and data written
+ // With the new fix, files should be directly under testPath, not in a "folder" subdirectory
+ expectedFiles := []string{
+ filepath.Join(testPath, "root_file.txt"),
+ filepath.Join(testPath, "nested", "deep_file.txt"),
+ }
+
+ for _, expectedFile := range expectedFiles {
+ if _, err := os.Stat(expectedFile); os.IsNotExist(err) {
+ t.Errorf("expected file %s to exist", expectedFile)
+ }
+ }
+ })
+
+ t.Run("No Items", func(t *testing.T) {
+ s, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, promutils.NewTestScope())
+ assert.NoError(t, err)
+
+ d := Downloader{store: s}
+
+ blob := &core.Blob{
+ Uri: "s3://container/folder",
+ Metadata: &core.BlobMetadata{
+ Type: &core.BlobType{
+ Dimensionality: core.BlobType_MULTIPART,
+ },
+ },
+ }
+
+ toPath := "./inputs"
+ defer func() {
+ err := os.RemoveAll(toPath)
+ if err != nil {
+ t.Errorf("Failed to delete directory: %v", err)
+ }
+ }()
+
+ result, err := d.handleBlob(context.Background(), blob, toPath)
+ assert.Error(t, err)
+ assert.Nil(t, result)
+ })
+}
+
+func TestHandleBlobSinglePart(t *testing.T) {
+ s, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, promutils.NewTestScope())
+ assert.NoError(t, err)
+ ref := storage.DataReference("s3://container/file")
+ err = s.WriteRaw(context.Background(), ref, 0, storage.Options{}, bytes.NewReader([]byte{}))
+ assert.NoError(t, err)
+
+ d := Downloader{store: s}
+
+ blob := &core.Blob{
+ Uri: "s3://container/file",
+ Metadata: &core.BlobMetadata{
+ Type: &core.BlobType{
+ Dimensionality: core.BlobType_SINGLE,
+ },
+ },
+ }
+
+ toPath := "./input"
+ defer func() {
+ err := os.RemoveAll(toPath)
+ if err != nil {
+ t.Errorf("Failed to delete file: %v", err)
+ }
+ }()
+
+ result, err := d.handleBlob(context.Background(), blob, toPath)
+ assert.NoError(t, err)
+ assert.Equal(t, toPath, result)
+
+ // Check if files were created and data written
+ if _, err := os.Stat(toPath); os.IsNotExist(err) {
+ t.Errorf("expected file %s to exist", toPath)
+ }
+}
+
+func TestHandleBlobHTTP(t *testing.T) {
+ s, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, promutils.NewTestScope())
+ assert.NoError(t, err)
+ d := Downloader{store: s}
+
+ blob := &core.Blob{
+ Uri: "https://raw.githubusercontent.com/flyteorg/flyte/master/README.md",
+ Metadata: &core.BlobMetadata{
+ Type: &core.BlobType{
+ Dimensionality: core.BlobType_SINGLE,
+ },
+ },
+ }
+
+ toPath := "./input"
+ defer func() {
+ err := os.RemoveAll(toPath)
+ if err != nil {
+ t.Errorf("Failed to delete file: %v", err)
+ }
+ }()
+
+ result, err := d.handleBlob(context.Background(), blob, toPath)
+ assert.NoError(t, err)
+ assert.Equal(t, toPath, result)
+
+ // Check if files were created and data written
+ if _, err := os.Stat(toPath); os.IsNotExist(err) {
+ t.Errorf("expected file %s to exist", toPath)
+ }
+}
+
+func TestRecursiveDownload(t *testing.T) {
+ t.Run("OffloadedMetadataContainsCollectionOfStrings", func(t *testing.T) {
+ s, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, promutils.NewTestScope())
+ assert.NoError(t, err)
+
+ d := Downloader{store: s}
+
+ offloadedLiteral := &core.Literal{
+ Value: &core.Literal_OffloadedMetadata{
+ OffloadedMetadata: &core.LiteralOffloadedMetadata{
+ Uri: "s3://container/offloaded",
+ },
+ },
+ }
+
+ inputs := &core.LiteralMap{
+ Literals: map[string]*core.Literal{
+ "input1": offloadedLiteral,
+ },
+ }
+
+ // Mock reading the offloaded metadata
+ err = s.WriteProtobuf(context.Background(), storage.DataReference("s3://container/offloaded"), storage.Options{}, &core.Literal{
+ Value: &core.Literal_Collection{
+ Collection: &core.LiteralCollection{
+ Literals: []*core.Literal{
+ {
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{
+ Primitive: &core.Primitive{
+ Value: &core.Primitive_StringValue{
+ StringValue: "string1",
+ },
+ },
+ },
+ },
+ },
+ },
+ {
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{
+ Primitive: &core.Primitive{
+ Value: &core.Primitive_StringValue{
+ StringValue: "string2",
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ })
+ assert.NoError(t, err)
+
+ toPath := "./inputs"
+ defer func() {
+ err := os.RemoveAll(toPath)
+ if err != nil {
+ t.Errorf("Failed to delete directory: %v", err)
+ }
+ }()
+
+ varMap, lMap, err := d.RecursiveDownload(context.Background(), inputs, toPath, true)
+ assert.NoError(t, err)
+ assert.NotNil(t, varMap)
+ assert.NotNil(t, lMap)
+ assert.Equal(t, []interface{}{"string1", "string2"}, varMap["input1"])
+ // Check if files were created and data written
+ for _, file := range []string{"0", "1"} {
+ if _, err := os.Stat(filepath.Join(toPath, "input1", file)); os.IsNotExist(err) {
+ t.Errorf("expected file %s to exist", file)
+ }
+ }
+ })
+
+ t.Run("OffloadedMetadataContainsMapOfStringString", func(t *testing.T) {
+ s, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, promutils.NewTestScope())
+ assert.NoError(t, err)
+
+ d := Downloader{store: s}
+
+ offloadedLiteral := &core.Literal{
+ Value: &core.Literal_OffloadedMetadata{
+ OffloadedMetadata: &core.LiteralOffloadedMetadata{
+ Uri: "s3://container/offloaded",
+ },
+ },
+ }
+
+ inputs := &core.LiteralMap{
+ Literals: map[string]*core.Literal{
+ "input1": offloadedLiteral,
+ },
+ }
+
+ // Mock reading the offloaded metadata
+ err = s.WriteProtobuf(context.Background(), storage.DataReference("s3://container/offloaded"), storage.Options{}, &core.Literal{
+ Value: &core.Literal_Map{
+ Map: &core.LiteralMap{
+ Literals: map[string]*core.Literal{
+ "key1": {
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{
+ Primitive: &core.Primitive{
+ Value: &core.Primitive_StringValue{
+ StringValue: "value1",
+ },
+ },
+ },
+ },
+ },
+ },
+ "key2": {
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{
+ Primitive: &core.Primitive{
+ Value: &core.Primitive_StringValue{
+ StringValue: "value2",
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ })
+ assert.NoError(t, err)
+
+ toPath := "./inputs"
+ defer func() {
+ err := os.RemoveAll(toPath)
+ if err != nil {
+ t.Errorf("Failed to delete directory: %v", err)
+ }
+ }()
+
+ varMap, lMap, err := d.RecursiveDownload(context.Background(), inputs, toPath, true)
+ assert.NoError(t, err)
+ assert.NotNil(t, varMap)
+ assert.NotNil(t, lMap)
+ assert.Equal(t, "value1", varMap["input1"].(VarMap)["key1"])
+ assert.Equal(t, "value2", varMap["input1"].(VarMap)["key2"])
+
+ for _, file := range []string{"key1", "key2"} {
+ if _, err := os.Stat(filepath.Join(toPath, "input1", file)); os.IsNotExist(err) {
+ t.Errorf("expected file %s to exist", file)
+ }
+ }
+ })
+}
+
+func TestHandleScalar(t *testing.T) {
+ t.Run("Handles Union Scalar with scalar value", func(t *testing.T) {
+ d := Downloader{}
+
+ scalar := &core.Scalar{
+ Value: &core.Scalar_Union{
+ Union: &core.Union{
+ Value: &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{
+ Primitive: &core.Primitive{
+ Value: &core.Primitive_StringValue{
+ StringValue: "string1",
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+
+ resultValue, resultScalar, err := d.handleScalar(context.Background(), scalar, "./inputs", false)
+ assert.NoError(t, err)
+ assert.Equal(t, "string1", resultValue)
+ assert.Equal(t, scalar, resultScalar)
+ })
+ t.Run("Handles Union Scalar with collection value", func(t *testing.T) {
+ d := Downloader{}
+
+ toPath := "./inputs"
+ defer func() {
+ err := os.RemoveAll(toPath)
+ if err != nil {
+ t.Errorf("Failed to delete directory: %v", err)
+ }
+ }()
+
+ scalar := &core.Scalar{
+ Value: &core.Scalar_Union{
+ Union: &core.Union{
+ Value: &core.Literal{
+ Value: &core.Literal_Collection{
+ Collection: &core.LiteralCollection{
+ Literals: []*core.Literal{
+ {
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{
+ Primitive: &core.Primitive{
+ Value: &core.Primitive_StringValue{
+ StringValue: "string1",
+ },
+ },
+ },
+ },
+ },
+ },
+ {
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{
+ Primitive: &core.Primitive{
+ Value: &core.Primitive_StringValue{
+ StringValue: "string2",
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+
+ resultValue, resultScalar, err := d.handleScalar(context.Background(), scalar, toPath, false)
+ assert.NoError(t, err)
+ assert.Equal(t, []interface{}{"string1", "string2"}, resultValue)
+ assert.Equal(t, scalar, resultScalar)
+ })
+}
diff --git a/flytecopilot/data/upload.go b/flytecopilot/data/upload.go
new file mode 100644
index 00000000000..b07ea52a37c
--- /dev/null
+++ b/flytecopilot/data/upload.go
@@ -0,0 +1,202 @@
+package data
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "path"
+ "path/filepath"
+ "reflect"
+
+ "github.com/golang/protobuf/proto"
+ "github.com/pkg/errors"
+
+ "github.com/flyteorg/flyte/v2/flyteidl2/clients/go/coreutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/futures"
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+const maxPrimitiveSize = 1024
+
+type Unmarshal func(r io.Reader, msg proto.Message) error
+type Uploader struct {
+ format core.DataLoadingConfig_LiteralMapFormat
+ mode core.IOStrategy_UploadMode
+ // TODO support multiple buckets
+ store *storage.DataStore
+ aggregateOutputFileName string
+ errorFileName string
+}
+
+type dirFile struct {
+ path string
+ info os.FileInfo
+ ref storage.DataReference
+}
+
+func (u Uploader) handleSimpleType(_ context.Context, t core.SimpleType, filePath string) (*core.Literal, error) {
+ fpath, info, err := IsFileReadable(filePath, true)
+ if err != nil {
+ return nil, err
+ }
+ if info.IsDir() {
+ return nil, fmt.Errorf("expected file for type [%s], found dir at path [%s]", t.String(), filePath)
+ }
+ if info.Size() > maxPrimitiveSize {
+ return nil, fmt.Errorf("maximum allowed filesize is [%d], but found [%d]", maxPrimitiveSize, info.Size())
+ }
+ b, err := ioutil.ReadFile(fpath)
+ if err != nil {
+ return nil, err
+ }
+ return coreutils.MakeLiteralForSimpleType(t, string(b))
+}
+
+func (u Uploader) handleBlobType(ctx context.Context, localPath string, toPath storage.DataReference) (*core.Literal, error) {
+ fpath, info, err := IsFileReadable(localPath, true)
+ if err != nil {
+ return nil, err
+ }
+ if info.IsDir() {
+ var files []dirFile
+ err := filepath.Walk(localPath, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ logger.Errorf(ctx, "encountered error when uploading multipart blob, %s", err)
+ return err
+ }
+ if info.IsDir() {
+ logger.Warnf(ctx, "Currently nested directories are not supported in multipart blob uploads, for directory @ %s", path)
+ } else {
+ ref, err := u.store.ConstructReference(ctx, toPath, info.Name())
+ if err != nil {
+ return err
+ }
+ files = append(files, dirFile{
+ path: path,
+ info: info,
+ ref: ref,
+ })
+ }
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ childCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ fileUploader := make([]futures.Future, 0, len(files))
+ for _, f := range files {
+ pth := f.path
+ ref := f.ref
+ size := f.info.Size()
+ fileUploader = append(fileUploader, futures.NewAsyncFuture(childCtx, func(i2 context.Context) (i interface{}, e error) {
+ return nil, UploadFileToStorage(i2, pth, ref, size, u.store)
+ }))
+ }
+
+ for _, f := range fileUploader {
+ // TODO maybe we should have timeouts, or we can have a global timeout at the top level
+ _, err := f.Get(ctx)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return coreutils.MakeLiteralForBlob(toPath, true, ""), nil
+ }
+ size := info.Size()
+ // Should we make this a go routine as well, so that we can introduce timeouts
+ return coreutils.MakeLiteralForBlob(toPath, false, ""), UploadFileToStorage(ctx, fpath, toPath, size, u.store)
+}
+
+func (u Uploader) RecursiveUpload(ctx context.Context, vars *core.VariableMap, fromPath string, metaOutputPath, dataRawPath storage.DataReference) error {
+ childCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
+ errFile := path.Join(fromPath, u.errorFileName)
+ if info, err := os.Stat(errFile); err != nil {
+ if !os.IsNotExist(err) {
+ return err
+ }
+ } else if info.Size() > 1024*1024 {
+ return fmt.Errorf("error file too large %d", info.Size())
+ } else if info.IsDir() {
+ return fmt.Errorf("error file is a directory")
+ } else {
+ b, err := ioutil.ReadFile(errFile)
+ if err != nil {
+ return err
+ }
+ return errors.Errorf("User Error: %s", string(b))
+ }
+
+ varFutures := make(map[string]futures.Future, len(vars.GetVariables()))
+ for _, variableEntry := range vars.GetVariables() {
+ varName := variableEntry.GetKey()
+ variable := variableEntry.GetValue()
+ varPath := path.Join(fromPath, varName)
+ varType := variable.GetType()
+ switch varType.GetType().(type) {
+ case *core.LiteralType_Blob:
+ var varOutputPath storage.DataReference
+ var err error
+ if varName == u.aggregateOutputFileName {
+ varOutputPath, err = u.store.ConstructReference(ctx, dataRawPath, "_"+varName)
+ } else {
+ varOutputPath, err = u.store.ConstructReference(ctx, dataRawPath, varName)
+ }
+ if err != nil {
+ return err
+ }
+ varFutures[varName] = futures.NewAsyncFuture(childCtx, func(ctx2 context.Context) (interface{}, error) {
+ return u.handleBlobType(ctx2, varPath, varOutputPath)
+ })
+ case *core.LiteralType_Simple:
+ varFutures[varName] = futures.NewAsyncFuture(childCtx, func(ctx2 context.Context) (interface{}, error) {
+ return u.handleSimpleType(ctx2, varType.GetSimple(), varPath)
+ })
+ default:
+ return fmt.Errorf("currently CoPilot uploader does not support [%s], system error", varType)
+ }
+ }
+
+ outputs := &core.LiteralMap{
+ Literals: make(map[string]*core.Literal, len(varFutures)),
+ }
+ for k, f := range varFutures {
+ logger.Infof(ctx, "Waiting for [%s] to complete (it may have a background upload too)", k)
+ v, err := f.Get(ctx)
+ if err != nil {
+ logger.Errorf(ctx, "Failed to upload [%s], reason [%s]", k, err)
+ return err
+ }
+ l, ok := v.(*core.Literal)
+ if !ok {
+ return fmt.Errorf("IllegalState, expected core.Literal, received [%s]", reflect.TypeOf(v))
+ }
+ outputs.Literals[k] = l
+ logger.Infof(ctx, "Var [%s] completed", k)
+ }
+
+ logger.Infof(ctx, "Uploading final outputs to [%s]", metaOutputPath)
+ if err := u.store.WriteProtobuf(ctx, metaOutputPath, storage.Options{}, outputs); err != nil {
+ logger.Errorf(ctx, "Failed to upload final outputs file to [%s], err [%s]", metaOutputPath, err)
+ return err
+ }
+ logger.Infof(ctx, "Uploaded final outputs to [%s]", metaOutputPath)
+ return nil
+}
+
+func NewUploader(_ context.Context, store *storage.DataStore, format core.DataLoadingConfig_LiteralMapFormat, mode core.IOStrategy_UploadMode, errorFileName string) Uploader {
+ return Uploader{
+ format: format,
+ store: store,
+ errorFileName: errorFileName,
+ mode: mode,
+ }
+}
diff --git a/flytecopilot/data/upload_test.go b/flytecopilot/data/upload_test.go
new file mode 100644
index 00000000000..c98b5c327cc
--- /dev/null
+++ b/flytecopilot/data/upload_test.go
@@ -0,0 +1,67 @@
+package data
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "os"
+ "path"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+func TestUploader_RecursiveUpload(t *testing.T) {
+
+ tmpFolderLocation := ""
+ tmpPrefix := "upload_test"
+
+ t.Run("upload-blob", func(t *testing.T) {
+ tmpDir, err := os.MkdirTemp(tmpFolderLocation, tmpPrefix)
+ assert.NoError(t, err)
+ defer func() {
+ assert.NoError(t, os.RemoveAll(tmpDir))
+ }()
+
+ vmap := &core.VariableMap{
+ Variables: []*core.VariableEntry{
+ {
+ Key: "x",
+ Value: &core.Variable{
+ Type: &core.LiteralType{Type: &core.LiteralType_Blob{Blob: &core.BlobType{Dimensionality: core.BlobType_SINGLE}}},
+ },
+ },
+ },
+ }
+
+ data := []byte("data")
+ assert.NoError(t, os.WriteFile(path.Join(tmpDir, "x"), data, os.ModePerm)) // #nosec G306
+ fmt.Printf("Written to %s ", path.Join(tmpDir, "x"))
+
+ store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, promutils.NewTestScope())
+ assert.NoError(t, err)
+
+ outputRef := storage.DataReference("output")
+ rawRef := storage.DataReference("raw")
+ u := NewUploader(context.TODO(), store, core.DataLoadingConfig_JSON, core.IOStrategy_UPLOAD_ON_EXIT, "error")
+ assert.NoError(t, u.RecursiveUpload(context.TODO(), vmap, tmpDir, outputRef, rawRef))
+
+ outputs := &core.LiteralMap{}
+ assert.NoError(t, store.ReadProtobuf(context.TODO(), outputRef, outputs))
+ assert.Len(t, outputs.GetLiterals(), 1)
+ assert.NotNil(t, outputs.GetLiterals()["x"])
+ assert.NotNil(t, outputs.GetLiterals()["x"].GetScalar())
+ assert.NotNil(t, outputs.GetLiterals()["x"].GetScalar().GetBlob())
+ ref := storage.DataReference(outputs.GetLiterals()["x"].GetScalar().GetBlob().GetUri())
+ r, err := store.ReadRaw(context.TODO(), ref)
+ assert.NoError(t, err, "%s does not exist", ref)
+ defer r.Close()
+ b, err := io.ReadAll(r)
+ assert.NoError(t, err)
+ assert.Equal(t, string(data), string(b), "content dont match")
+ })
+}
diff --git a/flytecopilot/data/utils.go b/flytecopilot/data/utils.go
new file mode 100644
index 00000000000..494c643bf01
--- /dev/null
+++ b/flytecopilot/data/utils.go
@@ -0,0 +1,91 @@
+package data
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "path/filepath"
+
+ "github.com/pkg/errors"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/logger"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+)
+
+// Checks if the given filepath is a valid and existing file path. If ignoreExtension is true, then the dir + basepath is checked for existence
+// ignoring the extension.
+// In the return the first return value is the actual path that exists (with the extension), second argument is the file info and finally the error
+func IsFileReadable(fpath string, ignoreExtension bool) (string, os.FileInfo, error) {
+ info, err := os.Stat(fpath)
+ if err != nil {
+ if os.IsNotExist(err) {
+ if ignoreExtension {
+ logger.Infof(context.TODO(), "looking for any extensions")
+ matches, err := filepath.Glob(fpath + ".*")
+ if err == nil && len(matches) == 1 {
+ logger.Infof(context.TODO(), "Extension match found [%s]", matches[0])
+ info, err = os.Stat(matches[0])
+ if err == nil {
+ return matches[0], info, nil
+ }
+ } else {
+ logger.Errorf(context.TODO(), "Extension match not found [%v,%v]", err, matches)
+ }
+ }
+ return "", nil, errors.Wrapf(err, "file not found at path [%s]", fpath)
+ }
+ if os.IsPermission(err) {
+ return "", nil, errors.Wrapf(err, "unable to read file [%s], Flyte does not have permissions", fpath)
+ }
+ return "", nil, errors.Wrapf(err, "failed to read file")
+ }
+ return fpath, info, nil
+}
+
+// Uploads a file to the data store.
+func UploadFileToStorage(ctx context.Context, filePath string, toPath storage.DataReference, size int64, store *storage.DataStore) error {
+ f, err := os.Open(filePath)
+ if err != nil {
+ return err
+ }
+ defer func() {
+ err := f.Close()
+ if err != nil {
+ logger.Errorf(ctx, "failed to close blob file at path [%s]", filePath)
+ }
+ }()
+ return store.WriteRaw(ctx, toPath, size, storage.Options{}, f)
+}
+
+func DownloadFileFromStorage(ctx context.Context, ref storage.DataReference, store *storage.DataStore) (io.ReadCloser, error) {
+ // We should probably directly use stow!??
+ m, err := store.Head(ctx, ref)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed when looking up Blob")
+ }
+ if m.Exists() {
+ r, err := store.ReadRaw(ctx, ref)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to read Blob from storage")
+ }
+ return r, err
+
+ }
+ return nil, fmt.Errorf("incorrect blob reference, does not exist")
+}
+
+// Downloads data from the given HTTP URL. If context is canceled then the request will be canceled.
+func DownloadFileFromHTTP(ctx context.Context, ref storage.DataReference) (io.ReadCloser, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, ref.String(), nil)
+ if err != nil {
+ logger.Errorf(ctx, "failed to create new http request with context, %s", err)
+ return nil, err
+ }
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return nil, errors.Wrapf(err, "Failed to download from url :%s", ref)
+ }
+ return resp.Body, nil
+}
diff --git a/flytecopilot/data/utils_test.go b/flytecopilot/data/utils_test.go
new file mode 100644
index 00000000000..5bf093d5cf1
--- /dev/null
+++ b/flytecopilot/data/utils_test.go
@@ -0,0 +1,119 @@
+package data
+
+import (
+ "bytes"
+ "context"
+ "os"
+ "path"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils"
+ "github.com/flyteorg/flyte/v2/flytestdlib/promutils/labeled"
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+)
+
+func TestIsFileReadable(t *testing.T) {
+ tmpFolderLocation := ""
+ tmpPrefix := "util_test"
+
+ tmpDir, err := os.MkdirTemp(tmpFolderLocation, tmpPrefix)
+ assert.NoError(t, err)
+ defer func() {
+ assert.NoError(t, os.RemoveAll(tmpDir))
+ }()
+ p := path.Join(tmpDir, "x")
+ f, i, err := IsFileReadable(p, false)
+ assert.Error(t, err)
+ assert.Empty(t, f)
+ assert.Nil(t, i)
+
+ assert.NoError(t, os.WriteFile(p, []byte("data"), os.ModePerm)) // #nosec G306
+ f, i, err = IsFileReadable(p, false)
+ assert.NoError(t, err)
+ assert.Equal(t, p, f)
+ assert.NotNil(t, i)
+ assert.Equal(t, p, f)
+
+ noExt := path.Join(tmpDir, "y")
+ p = path.Join(tmpDir, "y.png")
+ _, _, err = IsFileReadable(noExt, false)
+ assert.Error(t, err)
+
+ assert.NoError(t, os.WriteFile(p, []byte("data"), os.ModePerm)) // #nosec G306
+ _, _, err = IsFileReadable(noExt, false)
+ assert.Error(t, err)
+
+ f, i, err = IsFileReadable(noExt, true)
+ assert.NoError(t, err)
+ assert.Equal(t, p, f)
+ assert.NotNil(t, i)
+ assert.Equal(t, p, f)
+}
+
+func TestUploadFile(t *testing.T) {
+ tmpFolderLocation := ""
+ tmpPrefix := "util_test"
+
+ tmpDir, err := os.MkdirTemp(tmpFolderLocation, tmpPrefix)
+ assert.NoError(t, err)
+ defer func() {
+ assert.NoError(t, os.RemoveAll(tmpDir))
+ }()
+
+ exist := path.Join(tmpDir, "exist-file")
+ data := []byte("data")
+ l := int64(len(data))
+ assert.NoError(t, os.WriteFile(exist, data, os.ModePerm)) // #nosec G306
+ nonExist := path.Join(tmpDir, "non-exist-file")
+
+ store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, promutils.NewTestScope())
+ assert.NoError(t, err)
+
+ ctx := context.TODO()
+ assert.NoError(t, UploadFileToStorage(ctx, exist, "exist", l, store))
+ m, err := store.Head(ctx, "exist")
+ assert.True(t, m.Exists())
+ assert.NoError(t, err)
+
+ assert.Error(t, UploadFileToStorage(ctx, nonExist, "nonExist", l, store))
+}
+
+func TestDownloadFromHttp(t *testing.T) {
+ loc := storage.DataReference("https://raw.githubusercontent.com/flyteorg/flyte/master/README.md")
+ badLoc := storage.DataReference("https://no-exist")
+ f, err := DownloadFileFromHTTP(context.TODO(), loc)
+ if assert.NoError(t, err) {
+ if assert.NotNil(t, f) {
+ f.Close()
+ }
+ }
+
+ _, err = DownloadFileFromHTTP(context.TODO(), badLoc)
+ assert.Error(t, err)
+}
+
+func TestDownloadFromStorage(t *testing.T) {
+ store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, promutils.NewTestScope())
+ assert.NoError(t, err)
+ ref := storage.DataReference("ref")
+
+ f, err := DownloadFileFromStorage(context.TODO(), ref, store)
+ assert.Error(t, err)
+ assert.Nil(t, f)
+
+ data := []byte("data")
+ l := int64(len(data))
+
+ assert.NoError(t, store.WriteRaw(context.TODO(), ref, l, storage.Options{}, bytes.NewReader(data)))
+ f, err = DownloadFileFromStorage(context.TODO(), ref, store)
+ if assert.NoError(t, err) {
+ assert.NotNil(t, f)
+ f.Close()
+ }
+}
+
+func init() {
+ labeled.SetMetricKeys("test")
+}
diff --git a/flytecopilot/main.go b/flytecopilot/main.go
new file mode 100644
index 00000000000..5095b5771ab
--- /dev/null
+++ b/flytecopilot/main.go
@@ -0,0 +1,16 @@
+package main
+
+import (
+ "fmt"
+ "os"
+
+ "github.com/flyteorg/flyte/v2/flytecopilot/cmd"
+)
+
+func main() {
+ rootCmd := cmd.NewDataCommand()
+ if err := rootCmd.Execute(); err != nil {
+ fmt.Println(err)
+ os.Exit(1)
+ }
+}
diff --git a/flyteidl2/actions/actions_service.proto b/flyteidl2/actions/actions_service.proto
new file mode 100644
index 00000000000..e63ef49a248
--- /dev/null
+++ b/flyteidl2/actions/actions_service.proto
@@ -0,0 +1,200 @@
+syntax = "proto3";
+
+package flyteidl2.actions;
+
+import "buf/validate/validate.proto";
+import "flyteidl2/common/identifier.proto";
+import "flyteidl2/core/literals.proto";
+import "flyteidl2/task/run.proto";
+import "flyteidl2/workflow/run_definition.proto";
+import "flyteidl2/workflow/state_service.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/actions";
+
+// ActionsService provides an interface for managing the state and execution of actions.
+// This service is intended to replace StateService and QueueService, providing a single interface for managing both
+// the state and execution of actions.
+service ActionsService {
+ // Enqueue queues a new action for execution.
+ rpc Enqueue(EnqueueRequest) returns (EnqueueResponse) {}
+
+ // GetLatestState returns the latest `NodeStatus` of an action.
+ // This deprecates Get in the current StateService.
+ rpc GetLatestState(GetLatestStateRequest) returns (GetLatestStateResponse) {}
+
+ // WatchForUpdates watches for updates to the state of actions.
+ // This API guarantees at-least-once delivery semantics.
+ rpc WatchForUpdates(WatchForUpdatesRequest) returns (stream WatchForUpdatesResponse) {}
+
+ // Update updates the status of an action and saves serialized NodeStatus.
+ // This deprecates Put in the current StateService.
+ rpc Update(UpdateRequest) returns (UpdateResponse) {}
+
+ // Abort aborts a single action that was previously queued or is currently being processed by a worker.
+ // Note that this will cascade aborts to all descendant actions of the specified action.
+ rpc Abort(AbortRequest) returns (AbortResponse) {}
+
+ // Signal resolves a ConditionAction by providing its signal value.
+ // On success, transitions the condition to SUCCEEDED with the provided
+ // value as its output.
+ // Returns FAILED_PRECONDITION if the action is not a condition or is
+ // already terminal. Returns NOT_FOUND if the action does not exist.
+ // Returns ABORTED if the action has a write in-flight (retry).
+ rpc Signal(SignalRequest) returns (SignalResponse) {}
+}
+
+// Action represents a unit of work to be executed. Theses can be task executions, traces, or conditions.
+// Note: This is different from the Action protobuf defined in flyteidl2/workflow/run_definition.proto.
+message Action {
+ // the unique identifier for the action.
+ flyteidl2.common.ActionIdentifier action_id = 1 [(buf.validate.field).required = true];
+
+ // an optional name for the parent action, if it exists. the remaining run metadata (ex. org,
+ // project, domain) will be the same as the action_id defined above.
+ optional string parent_action_name = 2;
+
+ // the path to the input data for this action.
+ string input_uri = 3 [(buf.validate.field).string.min_len = 1];
+
+ // the run base path this action should write its output to.
+ string run_output_base = 4 [(buf.validate.field).string.min_len = 1];
+
+ // group this action belongs to, if applicable.
+ string group = 5;
+
+ // subject that created the run, if known.
+ string subject = 6;
+
+ oneof spec {
+ option (buf.validate.oneof).required = true;
+
+ // the spec for a task action. this will be required if the action is to execute a task.
+ flyteidl2.workflow.TaskAction task = 7;
+
+ // the spec for a trace action. this will be required if the action is to execute a trace.
+ flyteidl2.workflow.TraceAction trace = 8;
+
+ // the spec for a condition action. this will be required if the action is to execute a condition.
+ flyteidl2.workflow.ConditionAction condition = 9;
+ }
+}
+
+// ============================================================================
+// Enqueue
+// ============================================================================
+
+// EnqueueRequest is a request message for queuing an action.
+message EnqueueRequest {
+ Action action = 1 [(buf.validate.field).required = true];
+
+ // Optional run spec passed in by the root action to be utilized by all downstream actions in the run.
+ flyteidl2.task.RunSpec run_spec = 2;
+}
+
+// EnqueueResponse is a response message for queuing an action.
+message EnqueueResponse {}
+
+// ============================================================================
+// Update
+// ============================================================================
+
+// UpdateRequest is the request message for updating the status of an action.
+message UpdateRequest {
+ // A unique identifier for the action.
+ flyteidl2.common.ActionIdentifier action_id = 1 [(buf.validate.field).required = true];
+
+ // The attempt number (starts at 1, incremented on retry).
+ uint32 attempt = 2 [(buf.validate.field).uint32.gt = 0];
+
+ // The new status of the action.
+ flyteidl2.workflow.ActionStatus status = 3 [(buf.validate.field).required = true];
+
+ // The new state in form of a JSON serialized `NodeStatus` object.
+ string state = 4;
+}
+
+// UpdateResponse is the response message for updating the status of an action.
+message UpdateResponse {}
+
+// ============================================================================
+// GetLatestState
+// ============================================================================
+
+// GetLatestStateRequest is the request message for getting the state of an action.
+message GetLatestStateRequest {
+ // A unique identifier for the action.
+ flyteidl2.common.ActionIdentifier action_id = 1 [(buf.validate.field).required = true];
+
+ // The attempt number (starts at 1, incremented on retry).
+ uint32 attempt = 2 [(buf.validate.field).uint32.gt = 0];
+}
+
+// GetLatestStateResponse is the response message for getting the state of an action.
+message GetLatestStateResponse {
+ // A JSON serialized `NodeStatus` object.
+ string state = 1 [(buf.validate.field).string.min_len = 1];
+}
+
+// ============================================================================
+// Watch
+// ============================================================================
+
+// WatchForUpdatesRequest is a request message for watching updates to the state of actions.
+message WatchForUpdatesRequest {
+ // criteria for filtering which actions to watch.
+ oneof filter {
+ option (buf.validate.oneof).required = true;
+
+ // a unique identifier for the parent action to watch. this will result in updates for all child actions.
+ flyteidl2.common.ActionIdentifier parent_action_id = 1;
+ }
+}
+
+// WatchForUpdatesResponse is a response message for watching updates to the state of actions.
+message WatchForUpdatesResponse {
+ // an update to the state of a specific action.
+ oneof message {
+ // an update to the status of an action.
+ flyteidl2.workflow.ActionUpdate action_update = 1;
+
+ // a control message for the workflow execution.
+ flyteidl2.workflow.ControlMessage control_message = 2;
+ }
+}
+
+// ============================================================================
+// Abort
+// ============================================================================
+
+// AbortRequest is the request message for aborting a queued or running action.
+message AbortRequest {
+ // ActionId is the unique identifier for the action to be aborted
+ flyteidl2.common.ActionIdentifier action_id = 1 [(buf.validate.field).required = true];
+
+ // Reason for aborting the action, if applicable.
+ optional string reason = 2;
+}
+
+// AbortResponse is the response message for aborting a queued or running action.
+message AbortResponse {}
+
+// ============================================================================
+// Signal
+// ============================================================================
+
+// SignalRequest is the request message for resolving a condition action.
+message SignalRequest {
+ // The unique identifier for the condition action to signal.
+ flyteidl2.common.ActionIdentifier action_id = 1 [(buf.validate.field).required = true];
+
+ // The name of the parent action that owns this condition. Required for
+ // state store lookup and partition routing in the middleware.
+ string parent_action_name = 2 [(buf.validate.field).string.min_len = 1];
+
+ // The value literal to signal the condition with. Must match the
+ // ConditionAction.type declared at enqueue time.
+ flyteidl2.core.Literal value = 3 [(buf.validate.field).required = true];
+}
+
+// SignalResponse is the response message for resolving a condition action.
+message SignalResponse {}
diff --git a/flyteidl2/app/app_definition.proto b/flyteidl2/app/app_definition.proto
new file mode 100644
index 00000000000..0e21023a087
--- /dev/null
+++ b/flyteidl2/app/app_definition.proto
@@ -0,0 +1,444 @@
+syntax = "proto3";
+
+package flyteidl2.app;
+
+import "buf/validate/validate.proto";
+import "flyteidl2/common/identity.proto";
+import "flyteidl2/common/runtime_version.proto";
+import "flyteidl2/core/artifact_id.proto";
+import "flyteidl2/core/security.proto";
+import "flyteidl2/core/tasks.proto";
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app";
+
+message Identifier {
+ // Org that the app belongs to.
+ string org = 1;
+
+ // Project that the app belongs to.
+ string project = 2 [(buf.validate.field).string.min_len = 1];
+
+ // Domain that the app belongs to.
+ string domain = 3 [(buf.validate.field).string.min_len = 1];
+
+ // Name that the user provided for the app.
+ string name = 4 [
+ (buf.validate.field).string.min_len = 1,
+ (buf.validate.field).string.max_len = 30,
+ // Validate that the string is a valid DNS-1123 subdomain. This validation runs after
+ // the length validation, so it's safe to assume that the length is within the limits.
+ (buf.validate.field).string.pattern = "^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$"
+ ];
+}
+
+message Meta {
+ // ID of the app.
+ Identifier id = 1 [(buf.validate.field).required = true];
+
+ // Revision of the app object.
+ uint64 revision = 2 [(buf.validate.field).uint64.gte = 0];
+
+ // Labels for the app.
+ map labels = 3;
+}
+
+// For internal usage only. This message is used to wrap the app message with the host.
+message AppWrapper {
+ string host = 1;
+ oneof payload {
+ App app = 2;
+ Identifier app_id = 3;
+ }
+}
+
+// Represents an app with its specification and status.
+message App {
+ // Metadata of the app.
+ Meta metadata = 1 [(buf.validate.field).required = true];
+
+ // Specification of the app.
+ Spec spec = 2 [(buf.validate.field).required = true];
+
+ // Status of the app.
+ Status status = 3;
+}
+
+/* State machine for the app status
+
+ ┌──────────────┐
+ │ │
+ ┌──────────► Unassigned │
+ │ │ │
+ │ └──────┬───────┘
+ │Lease │
+ │Expired │ Leased to a cluster
+ │ │
+ │ ┌──────▼───────┐
+ │ │ Assigned │
+ └──────────┼ + │
+ │ Timeout │
+ └──────┬───────┘
+ │
+ │ Acknowledged by cluster
+ │
+ ┌──────▼───────┐
+ ┌──────────► Pending │
+ │ └──────┬───────┘
+ │New │
+ │Revision │ Action performed by cluster
+ │ │
+ │ ┌───────────▼────────────┐
+ └─────┼ Stopped/Started/Failed │
+ └────────────────────────┘
+*/
+
+// Represents the condition of an app.
+message Condition {
+ // Last transition time of the condition.
+ google.protobuf.Timestamp last_transition_time = 1;
+
+ // Deployment status of the app.
+ Status.DeploymentStatus deployment_status = 2;
+
+ // Message for the condition.
+ string message = 3;
+
+ // Revision the Condition applies to. This can be used by consumers
+ // to introspect and visualize which revisions are up.
+ uint64 revision = 4 [(buf.validate.field).uint64.gte = 0];
+
+ // Actor is the principal that caused the condition.
+ common.EnrichedIdentity actor = 5;
+}
+
+// Represents the status of an app.
+message Status {
+ string assigned_cluster = 1;
+
+ // Enum for deployment status of the app.
+ enum DeploymentStatus {
+ // Unspecified deployment status.
+ DEPLOYMENT_STATUS_UNSPECIFIED = 0;
+ // Deployment is enabled but hasn't been assigned to a cluster yet.
+ DEPLOYMENT_STATUS_UNASSIGNED = 1;
+ // Deployment is assigned to a cluster but hasn't been acknowledged yet.
+ DEPLOYMENT_STATUS_ASSIGNED = 2;
+ // Deployment is picked up by a cluster but is awaiting deployment.
+ DEPLOYMENT_STATUS_PENDING = 3;
+ // Deployment is disabled.
+ DEPLOYMENT_STATUS_STOPPED = 4;
+ // Deployment is completed. Please use DEPLOYMENT_STATUS_ACTIVE instead.
+ DEPLOYMENT_STATUS_STARTED = 5 [deprecated = true];
+ // Deployment has failed.
+ DEPLOYMENT_STATUS_FAILED = 6;
+ // Deployment is completed.
+ DEPLOYMENT_STATUS_ACTIVE = 7;
+ // Triggered in response to desired app replicas > actual app replicas
+ DEPLOYMENT_STATUS_SCALING_UP = 8;
+ // Triggered in response to desired app replicas < actual app replicas
+ DEPLOYMENT_STATUS_SCALING_DOWN = 9;
+ // Triggered in response to the latest app spec changing
+ DEPLOYMENT_STATUS_DEPLOYING = 10;
+ }
+
+ // Current number of replicas.
+ uint32 current_replicas = 2 [(buf.validate.field).uint32.gte = 0];
+
+ // List of public URLs for the app.
+ Ingress ingress = 3;
+
+ // CreatedAt is the time when the app was first created
+ google.protobuf.Timestamp created_at = 4;
+
+ // LastUpdatedAt is the time when the app was last updated
+ google.protobuf.Timestamp last_updated_at = 5;
+
+ // Conditions for the app.
+ repeated Condition conditions = 6;
+
+ // LeaseExpiration refers to how long the app is leased to a cluster for it to start processing it. If the lease
+ // period expires, the cluster will no longer be allowed to update this app and another cluster can pick it up.
+ google.protobuf.Timestamp lease_expiration = 7;
+
+ // K8sMetadata contains metadata about the app in the K8s cluster.
+ K8sMetadata k8s_metadata = 8;
+
+ MaterializedInputs materialized_inputs = 9;
+}
+
+message K8sMetadata {
+ // Namespace points to the namespace the app is deployed in.
+ string namespace = 1;
+}
+
+message Ingress {
+ // Public URL of the app.
+ string public_url = 1 [
+ (buf.validate.field).ignore = IGNORE_IF_ZERO_VALUE,
+ (buf.validate.field).string.uri = true
+ ];
+
+ // Canonical name (CNAME) URL of the app.
+ string cname_url = 2 [
+ (buf.validate.field).ignore = IGNORE_IF_ZERO_VALUE,
+ (buf.validate.field).string.uri = true
+ ];
+
+ // VPC URL of the app.
+ string vpc_url = 3 [
+ (buf.validate.field).ignore = IGNORE_IF_ZERO_VALUE,
+ (buf.validate.field).string.uri = true
+ ];
+}
+
+// Represents the specification of an app.
+message Spec {
+ // Payload of the app, which can be either a container or a K8s pod.
+ oneof app_payload {
+ option (buf.validate.oneof).required = true;
+
+ // Container payload.
+ flyteidl2.core.Container container = 1;
+
+ // K8s pod payload.
+ flyteidl2.core.K8sPod pod = 2;
+ }
+
+ // Autoscaling configuration for the app.
+ AutoscalingConfig autoscaling = 3;
+
+ // Ingress configuration for the app.
+ IngressConfig ingress = 4;
+
+ // Enum for deployment status of the app.
+ enum DesiredState {
+ // Unspecified state
+ DESIRED_STATE_UNSPECIFIED = 0;
+ // Deployment is disabled.
+ DESIRED_STATE_STOPPED = 1;
+ // Deployment is completed. Please use DESIRED_STATE_ACTIVE instead.
+ DESIRED_STATE_STARTED = 2 [deprecated = true];
+ // Deployment is completed.
+ DESIRED_STATE_ACTIVE = 3;
+ }
+
+ // Deployment status of the app.
+ DesiredState desired_state = 5;
+
+ // ClusterPool to place this app on. By default it'll use the default cluster pool.
+ string cluster_pool = 6;
+
+ // Set of image specifications for the app.
+ ImageSpecSet images = 7;
+
+ // security_context encapsulates security attributes requested to run this task.
+ SecurityContext security_context = 8;
+
+ // Encapsulates all non-standard resources, not captured by
+ // v1.ResourceRequirements, to allocate to a task.
+ flyteidl2.core.ExtendedResources extended_resources = 9;
+
+ // Runtime metadata for the app.
+ common.RuntimeMetadata runtime_metadata = 10;
+
+ // Profile of the app.
+ Profile profile = 11;
+
+ // Creator of the app is the first principal that provisioned this app. Other principals may
+ // interact with the app by updating its spec or stopping/starting it.
+ common.EnrichedIdentity creator = 12;
+
+ // Inputs of the app.
+ InputList inputs = 13;
+
+ repeated Link links = 14;
+
+ // Timeout configuration for the app.
+ TimeoutConfig timeouts = 15;
+}
+
+// Represents a link to an external resource (Arize project link, W&B dashboard, etc)
+message Link {
+ // URL of the external service.
+ // This can be an absolute or relative path.
+ string path = 1 [(buf.validate.field).string.min_len = 1];
+
+ // Human readable name of the external service.
+ string title = 2;
+
+ // Whether the path is absolute (default) or relative.
+ bool is_relative = 3;
+}
+
+message Input {
+ // Name is a unique identifier of the input within the list of inputs of the app
+ string name = 1 [(buf.validate.field).string.min_len = 1];
+
+ oneof value {
+ option (buf.validate.oneof).required = true;
+
+ // StringValue is a plain string value for the input
+ string string_value = 2 [(buf.validate.field).string.min_len = 1];
+
+ // ArtifactQuery is that should result in a single artifact that will be used as the input to the app at runtime.
+ // The artifact will be pinned at the time of the app creation.
+ flyteidl2.core.ArtifactQuery artifact_query = 3;
+
+ // ArtifactId is an identifier of an artifact that will be used as the input to the app at runtime.
+ flyteidl2.core.ArtifactID artifact_id = 4;
+
+ // ID of the app.
+ Identifier app_id = 5;
+ }
+}
+
+message MaterializedInputs {
+ repeated MaterializedInput items = 1;
+
+ // Revision of the app object that we materialized the input for.
+ uint64 revision = 2 [(buf.validate.field).uint64.gte = 0];
+}
+
+message MaterializedInput {
+ // Name is a unique identifier of the input within the list of inputs of the app
+ string name = 1 [(buf.validate.field).string.min_len = 1];
+
+ oneof value {
+ option (buf.validate.oneof).required = true;
+
+ // ArtifactId is an identifier of an artifact that will be used as the input to the app at runtime.
+ flyteidl2.core.ArtifactID artifact_id = 2;
+ }
+}
+
+// InputList is a list of dependencies for the app.
+message InputList {
+ // Items is the list of inputs to fulfil for the app.
+ repeated Input items = 1;
+}
+
+message Profile {
+ // App Type (e.g. FastAPI, Flask, VLLM, NIM etc.)
+ string type = 1;
+
+ // Friendly name of the app.
+ string name = 2;
+
+ // Short description of the app.
+ string short_description = 3 [(buf.validate.field).string.max_len = 100];
+
+ // Icon URL of the app.
+ string icon_url = 4;
+}
+
+// SecurityContext holds security attributes that apply to tasks.
+message SecurityContext {
+ // run_as encapsulates the identity a pod should run as. If the task fills in multiple fields here, it'll be up to the
+ // backend plugin to choose the appropriate identity for the execution engine the task will run on.
+ flyteidl2.core.Identity run_as = 1;
+
+ // secrets indicate the list of secrets the task needs in order to proceed. Secrets will be mounted/passed to the
+ // pod as it starts. If the plugin responsible for kicking of the task will not run it on a flyte cluster (e.g. AWS
+ // Batch), it's the responsibility of the plugin to fetch the secret (which means propeller identity will need access
+ // to the secret) and to pass it to the remote execution engine.
+ repeated flyteidl2.core.Secret secrets = 2;
+
+ reserved 3;
+ reserved 4;
+
+ // AllowAnonymous indicates if the app should be accessible without authentication. This assumes the app will handle
+ // its own authentication or that it's a public app.
+ bool allow_anonymous = 5;
+}
+
+message ImageSpec {
+ // Tag of the image.
+ string tag = 1;
+ // URL of the build job for the image.
+ string build_job_url = 2;
+}
+
+message ImageSpecSet {
+ // List of image specifications.
+ repeated ImageSpec images = 1;
+}
+
+// Represents the ingress configuration of an app.
+message IngressConfig {
+ // Indicates if the app should be private.
+ bool private = 1;
+
+ // Subdomain for the app. If not specified, a random subdomain will be generated.
+ string subdomain = 2;
+
+ // Canonical name (CNAME) for the app.
+ string cname = 3;
+}
+
+// Represents the autoscaling configuration of an app.
+message AutoscalingConfig {
+ // Configuration for the number of replicas.
+ Replicas replicas = 1;
+ // The period for scaling down the object.
+ google.protobuf.Duration scaledown_period = 2;
+ // Metric for scaling the app.
+ ScalingMetric scaling_metric = 3;
+}
+
+// ScalingMetric allows different scaling strategies for the app.
+// See: https://knative.dev/docs/serving/autoscaling/autoscaling-metrics/
+message ScalingMetric {
+ oneof metric {
+ option (buf.validate.oneof).required = true;
+
+ // Configuration for scaling based on request rate.
+ RequestRate request_rate = 1;
+ // Configuration for scaling based on concurrency.
+ Concurrency concurrency = 2;
+ }
+}
+
+// This section enables scaling based on the request concurrency. Concurrency calculates how many
+// requests are being processed at the same time. This is useful for apps that take longer to process
+// requests (e.g. seconds)
+// The autoscaler has a default window of 60 seconds to calculate the concurrency value. However, it has
+// panic mode that kicks in if the request rate jumps to 2x its value within 6 seconds. If that's
+// triggered, it'll start scaling up immediately instead of waiting for the full 60 seconds.
+message Concurrency {
+ // This is the target value for the scaling configuration.
+ // default=100
+ uint32 target_value = 1 [(buf.validate.field).uint32.gt = 0];
+}
+
+// RequestRate enables scaling based on the request rate. Request rate calculates how many requests
+// the app is receiving per second.
+// The autoscaler has a default window of 60 seconds to calculate the request rate. However, it has
+// panic mode that kicks in if the request rate jumps to 2x its value within 6 seconds. If that's
+// triggered, it'll start scaling up immediately instead of waiting for the full 60 seconds.
+message RequestRate {
+ // This is the target value for the scaling configuration.
+ // default=100
+ uint32 target_value = 1 [(buf.validate.field).uint32.gte = 0];
+}
+
+// Represents the configuration for the number of replicas.
+message Replicas {
+ // Minimum number of replicas.
+ uint32 min = 1 [(buf.validate.field).uint32.gte = 0];
+
+ // Maximum number of replicas.
+ uint32 max = 2 [(buf.validate.field).uint32.gte = 0];
+}
+
+message TimeoutConfig {
+ // This is the maximum duration that the request instance
+ // is allowed to respond to a request. If unspecified, a system default will
+ // be provided.
+ // default=300s
+ google.protobuf.Duration request_timeout = 1 [(buf.validate.field).duration = {
+ gte: {seconds: 0}
+ lte: {seconds: 3600}
+ }];
+}
diff --git a/flyteidl2/app/app_logs_payload.proto b/flyteidl2/app/app_logs_payload.proto
new file mode 100644
index 00000000000..cb22381e546
--- /dev/null
+++ b/flyteidl2/app/app_logs_payload.proto
@@ -0,0 +1,56 @@
+syntax = "proto3";
+
+package flyteidl2.app;
+
+import "buf/validate/validate.proto";
+import "flyteidl2/app/app_definition.proto";
+import "flyteidl2/app/replica_definition.proto";
+import "flyteidl2/logs/dataplane/payload.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app";
+
+message TailLogsRequest {
+ oneof target {
+ option (buf.validate.oneof).required = true;
+
+ // Identifier of the application to get logs for.
+ Identifier app_id = 1;
+
+ // Identifier of the replica to get logs for.
+ ReplicaIdentifier replica_id = 2;
+ }
+}
+
+message ReplicaIdentifierList {
+ repeated ReplicaIdentifier replicas = 1;
+}
+
+message LogLines {
+ repeated string lines = 1 [deprecated = true];
+
+ ReplicaIdentifier replica_id = 2 [(buf.validate.field).required = true];
+
+ repeated flyteidl2.logs.dataplane.LogLine structured_lines = 3;
+}
+
+message LogLinesBatch {
+ repeated LogLines logs = 1;
+}
+
+message TailLogsResponse {
+ oneof resp {
+ option (buf.validate.oneof).required = true;
+
+ // Replicas lists the replicas that the logs are being tailed for. This is expected to be the first
+ // message to be sent in the stream but also can be sent at any later time to update the list of
+ // replicas being tailed.
+ ReplicaIdentifierList replicas = 1;
+
+ // The latest log lines for the application.
+ // Deprecated, use batches instead.
+ flyteidl2.logs.dataplane.LogLines log_lines = 2 [deprecated = true];
+
+ // The latest log lines for the application.
+ LogLinesBatch batches = 3;
+ }
+}
diff --git a/flyteidl2/app/app_logs_service.proto b/flyteidl2/app/app_logs_service.proto
new file mode 100644
index 00000000000..445ac124313
--- /dev/null
+++ b/flyteidl2/app/app_logs_service.proto
@@ -0,0 +1,13 @@
+syntax = "proto3";
+
+package flyteidl2.app;
+
+import "flyteidl2/app/app_logs_payload.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app";
+
+service AppLogsService {
+ rpc TailLogs(TailLogsRequest) returns (stream TailLogsResponse) {
+ option idempotency_level = NO_SIDE_EFFECTS;
+ }
+}
diff --git a/flyteidl2/app/app_payload.proto b/flyteidl2/app/app_payload.proto
new file mode 100644
index 00000000000..9fc54bf67ee
--- /dev/null
+++ b/flyteidl2/app/app_payload.proto
@@ -0,0 +1,162 @@
+syntax = "proto3";
+
+package flyteidl2.app;
+
+import "buf/validate/validate.proto";
+import "flyteidl2/app/app_definition.proto";
+import "flyteidl2/common/identifier.proto";
+import "flyteidl2/common/list.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app";
+
+// Request message for creating an app.
+message CreateRequest {
+ // The app to be created.
+ App app = 1 [(buf.validate.field).required = true];
+}
+
+// Response message for creating an app.
+message CreateResponse {
+ // The created app.
+ App app = 1;
+}
+
+// Request message for retrieving an app.
+message GetRequest {
+ oneof identifier {
+ option (buf.validate.oneof).required = true;
+
+ // Identifier of the app to be retrieved.
+ Identifier app_id = 1;
+
+ // Ingress of the app to be retrieved. Only one field need to be set.
+ // If multiple fields are set, they must resolve into the same app.
+ // Otherwise, an error is returned.
+ Ingress ingress = 2;
+ }
+}
+
+// Response message for retrieving an app.
+message GetResponse {
+ // The retrieved app.
+ App app = 1;
+}
+
+// Request message for updating an app.
+message UpdateRequest {
+ // The app to be updated.
+ App app = 1 [(buf.validate.field).required = true];
+ string reason = 2 [(buf.validate.field).string.max_len = 100];
+}
+
+// Response message for updating an app.
+message UpdateResponse {
+ // The updated app.
+ App app = 1;
+}
+
+// Request message for deleting an app.
+message DeleteRequest {
+ // Identifier of the app to be deleted.
+ Identifier app_id = 1 [(buf.validate.field).required = true];
+}
+
+// Response message for deleting an app.
+message DeleteResponse {}
+
+// Request message for listing apps.
+message ListRequest {
+ // Common list request parameters.
+ common.ListRequest request = 1;
+
+ oneof filter_by {
+ option (buf.validate.oneof).required = true;
+
+ // Organization name for filtering apps.
+ string org = 2 [(buf.validate.field).string.min_len = 1];
+ // Cluster identifier for filtering apps.
+ common.ClusterIdentifier cluster_id = 3;
+ // Project identifier for filtering apps.
+ common.ProjectIdentifier project = 4;
+ }
+}
+
+// Response message for listing apps.
+message ListResponse {
+ // List of apps.
+ repeated App apps = 1;
+ // Token for fetching the next page of results, if any.
+ string token = 2;
+}
+
+// Request message for watching app events.
+message WatchRequest {
+ oneof target {
+ option (buf.validate.oneof).required = true;
+
+ // Organization name for filtering events.
+ string org = 1 [(buf.validate.field).string.min_len = 1];
+ // Cluster identifier for filtering events.
+ common.ClusterIdentifier cluster_id = 2;
+ // Project identifier for filtering events.
+ common.ProjectIdentifier project = 3;
+ // App identifier for filtering events.
+ Identifier app_id = 4;
+ }
+}
+
+// Event message for app creation.
+message CreateEvent {
+ // The created app.
+ App app = 1;
+}
+
+// Event message for app update.
+message UpdateEvent {
+ // The updated app.
+ App updated_app = 1;
+ // The old app before the update.
+ App old_app = 2;
+}
+
+// Event message for app deletion.
+message DeleteEvent {
+ // The deleted app.
+ App app = 1;
+}
+
+// Response message for watching app events.
+message WatchResponse {
+ oneof event {
+ // Event for app creation.
+ CreateEvent create_event = 1;
+ // Event for app update.
+ UpdateEvent update_event = 2;
+ // Event for app deletion.
+ DeleteEvent delete_event = 3;
+ }
+}
+
+// Request message for updating app status.
+message UpdateStatusRequest {
+ // The app with updated status.
+ App app = 1 [(buf.validate.field).required = true];
+}
+
+// Response message for updating app status.
+message UpdateStatusResponse {
+ // The app with updated status.
+ App app = 1;
+}
+
+// Request message for leasing apps.
+message LeaseRequest {
+ // Cluster identifier for leasing apps.
+ common.ClusterIdentifier id = 1 [(buf.validate.field).required = true];
+}
+
+// Response message for leasing apps.
+message LeaseResponse {
+ // List of leased apps.
+ repeated App apps = 1;
+}
diff --git a/flyteidl2/app/app_service.proto b/flyteidl2/app/app_service.proto
new file mode 100644
index 00000000000..443577c8cae
--- /dev/null
+++ b/flyteidl2/app/app_service.proto
@@ -0,0 +1,40 @@
+syntax = "proto3";
+
+package flyteidl2.app;
+
+import "flyteidl2/app/app_payload.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app";
+
+// AppService defines the service for managing apps.
+service AppService {
+ // Create creates a new app.
+ rpc Create(CreateRequest) returns (CreateResponse) {}
+
+ // Get retrieves an app by its identifier.
+ rpc Get(GetRequest) returns (GetResponse) {
+ option idempotency_level = NO_SIDE_EFFECTS;
+ }
+
+ // Update updates an existing app.
+ rpc Update(UpdateRequest) returns (UpdateResponse) {}
+
+ // UpdateStatus updates the status of an existing app.
+ rpc UpdateStatus(UpdateStatusRequest) returns (UpdateStatusResponse) {}
+
+ // Delete deletes an app by its identifier.
+ rpc Delete(DeleteRequest) returns (DeleteResponse) {}
+
+ // List lists all apps with optional filtering.
+ rpc List(ListRequest) returns (ListResponse) {
+ option idempotency_level = NO_SIDE_EFFECTS;
+ }
+
+ // Watch watches for app events.
+ rpc Watch(WatchRequest) returns (stream WatchResponse) {
+ option idempotency_level = NO_SIDE_EFFECTS;
+ }
+
+ // Lease leases apps.
+ rpc Lease(LeaseRequest) returns (stream LeaseResponse) {}
+}
diff --git a/flyteidl2/app/replica_definition.proto b/flyteidl2/app/replica_definition.proto
new file mode 100644
index 00000000000..44241074b34
--- /dev/null
+++ b/flyteidl2/app/replica_definition.proto
@@ -0,0 +1,43 @@
+syntax = "proto3";
+
+package flyteidl2.app;
+
+import "buf/validate/validate.proto";
+import "flyteidl2/app/app_definition.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/app";
+
+message ReplicaIdentifier {
+ // ID of the app.
+ Identifier app_id = 1 [(buf.validate.field).required = true];
+ string name = 2 [(buf.validate.field).string.min_len = 1];
+}
+
+message ReplicaMeta {
+ // ID of the replica.
+ ReplicaIdentifier id = 1 [(buf.validate.field).required = true];
+
+ // Revision of the replica object.
+ uint64 revision = 2;
+}
+
+message ReplicaList {
+ // List of replicas.
+ repeated Replica items = 1;
+}
+
+// Represents a replica of an app with its specification and status.
+message Replica {
+ // Metadata of the app.
+ ReplicaMeta metadata = 1 [(buf.validate.field).required = true];
+ // Status of the app.
+ ReplicaStatus status = 2;
+}
+
+// Represents the status of the replica.
+message ReplicaStatus {
+ // Current deployment status of the replica.
+ string deployment_status = 1;
+
+ string reason = 2;
+}
diff --git a/flyteidl2/auth/auth_service.proto b/flyteidl2/auth/auth_service.proto
new file mode 100644
index 00000000000..4c45e1a1ad2
--- /dev/null
+++ b/flyteidl2/auth/auth_service.proto
@@ -0,0 +1,83 @@
+syntax = "proto3";
+package flyteidl2.auth;
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/auth";
+
+message GetOAuth2MetadataRequest {}
+
+// OAuth2MetadataResponse defines an RFC-Compliant response for /.well-known/oauth-authorization-server metadata
+// as defined in https://tools.ietf.org/html/rfc8414
+message GetOAuth2MetadataResponse {
+ // Defines the issuer string in all JWT tokens this server issues. The issuer can be admin itself or an external
+ // issuer.
+ string issuer = 1;
+
+ // URL of the authorization server's authorization endpoint [RFC6749]. This is REQUIRED unless no grant types are
+ // supported that use the authorization endpoint.
+ string authorization_endpoint = 2;
+
+ // URL of the authorization server's token endpoint [RFC6749].
+ string token_endpoint = 3;
+
+ // Array containing a list of the OAuth 2.0 response_type values that this authorization server supports.
+ repeated string response_types_supported = 4;
+
+ // JSON array containing a list of the OAuth 2.0 [RFC6749] scope values that this authorization server supports.
+ repeated string scopes_supported = 5;
+
+ // JSON array containing a list of client authentication methods supported by this token endpoint.
+ repeated string token_endpoint_auth_methods_supported = 6;
+
+ // URL of the authorization server's JWK Set [JWK] document. The referenced document contains the signing key(s) the
+ // client uses to validate signatures from the authorization server.
+ string jwks_uri = 7;
+
+ // JSON array containing a list of Proof Key for Code Exchange (PKCE) [RFC7636] code challenge methods supported by
+ // this authorization server.
+ repeated string code_challenge_methods_supported = 8;
+
+ // JSON array containing a list of the OAuth 2.0 grant type values that this authorization server supports.
+ repeated string grant_types_supported = 9;
+
+ // URL of the authorization server's device authorization endpoint, as defined in Section 3.1 of [RFC8628]
+ string device_authorization_endpoint = 10;
+}
+
+message GetPublicClientConfigRequest {}
+
+// GetPublicClientConfigResponse encapsulates public information that flyte clients (CLIs... etc.) can use to authenticate users.
+message GetPublicClientConfigResponse {
+ // client_id to use when initiating OAuth2 authorization requests.
+ string client_id = 1;
+ // redirect uri to use when initiating OAuth2 authorization requests.
+ string redirect_uri = 2;
+ // scopes to request when initiating OAuth2 authorization requests.
+ repeated string scopes = 3;
+ // Authorization Header to use when passing Access Tokens to the server. If not provided, the client should use the
+ // default http `Authorization` header.
+ string authorization_metadata_key = 4;
+ // ServiceHttpEndpoint points to the http endpoint for the backend. If empty, clients can assume the endpoint used
+ // to configure the gRPC connection can be used for the http one respecting the insecure flag to choose between
+ // SSL or no SSL connections.
+ string service_http_endpoint = 5;
+ // audience to use when initiating OAuth2 authorization requests.
+ string audience = 6;
+ // Returns the endpoint to use for dataplane specific operations.
+ string dataplane_domain = 7;
+}
+
+// The following defines an RPC service that is also served over HTTP via grpc-gateway.
+// Standard response codes for both are defined here: https://github.com/grpc-ecosystem/grpc-gateway/blob/master/runtime/errors.go
+// RPCs defined in this service must be anonymously accessible.
+service AuthMetadataService {
+ // Anonymously accessible. Retrieves local or external oauth authorization server metadata.
+ rpc GetOAuth2Metadata(GetOAuth2MetadataRequest) returns (GetOAuth2MetadataResponse) {
+ option idempotency_level = NO_SIDE_EFFECTS;
+ }
+
+ // Anonymously accessible. Retrieves the client information clients should use when initiating OAuth2 authorization
+ // requests.
+ rpc GetPublicClientConfig(GetPublicClientConfigRequest) returns (GetPublicClientConfigResponse) {
+ option idempotency_level = NO_SIDE_EFFECTS;
+ }
+}
diff --git a/flyteidl2/auth/identity.proto b/flyteidl2/auth/identity.proto
new file mode 100644
index 00000000000..8c9365bdc64
--- /dev/null
+++ b/flyteidl2/auth/identity.proto
@@ -0,0 +1,42 @@
+syntax = "proto3";
+package flyteidl2.auth;
+
+import "google/protobuf/struct.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/auth";
+
+message UserInfoRequest {}
+
+// See the OpenID Connect spec at https://openid.net/specs/openid-connect-core-1_0.html#UserInfoResponse for more information.
+message UserInfoResponse {
+ // Locally unique and never reassigned identifier within the Issuer for the End-User, which is intended to be consumed
+ // by the Client.
+ string subject = 1;
+
+ // Full name
+ string name = 2;
+
+ // Shorthand name by which the End-User wishes to be referred to
+ string preferred_username = 3;
+
+ // Given name(s) or first name(s)
+ string given_name = 4;
+
+ // Surname(s) or last name(s)
+ string family_name = 5;
+
+ // Preferred e-mail address
+ string email = 6;
+
+ // Profile picture URL
+ string picture = 7;
+
+ // Additional claims
+ google.protobuf.Struct additional_claims = 8;
+}
+
+// IdentityService defines an RPC Service that interacts with user/app identities.
+service IdentityService {
+ // Retrieves user information about the currently logged in user.
+ rpc UserInfo(UserInfoRequest) returns (UserInfoResponse) {}
+}
diff --git a/flyteidl2/cacheservice/cacheservice.proto b/flyteidl2/cacheservice/cacheservice.proto
new file mode 100644
index 00000000000..efbfef105b0
--- /dev/null
+++ b/flyteidl2/cacheservice/cacheservice.proto
@@ -0,0 +1,147 @@
+syntax = "proto3";
+
+package flyteidl2.cacheservice;
+
+import "flyteidl2/core/identifier.proto";
+import "flyteidl2/core/literals.proto";
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/cacheservice";
+
+/*
+ * CacheService defines operations for cache management including retrieval, storage, and deletion of cached task/workflow outputs.
+ */
+service CacheService {
+ // Retrieves cached data by key.
+ rpc Get(GetCacheRequest) returns (GetCacheResponse);
+
+ // Stores or updates cached data by key.
+ rpc Put(PutCacheRequest) returns (PutCacheResponse);
+
+ // Deletes cached data by key.
+ rpc Delete(DeleteCacheRequest) returns (DeleteCacheResponse);
+
+ // Get or extend a reservation for a cache key
+ rpc GetOrExtendReservation(GetOrExtendReservationRequest) returns (GetOrExtendReservationResponse);
+
+ // Release the reservation for a cache key
+ rpc ReleaseReservation(ReleaseReservationRequest) returns (ReleaseReservationResponse);
+}
+
+/*
+ * Additional metadata as key-value pairs
+ */
+message KeyMapMetadata {
+ map values = 1; // Additional metadata as key-value pairs
+}
+
+/*
+ * Metadata for cached outputs, including the source identifier and timestamps.
+ */
+message Metadata {
+ core.Identifier source_identifier = 1; // Source task or workflow identifier
+ KeyMapMetadata key_map = 2; // Additional metadata as key-value pairs
+ google.protobuf.Timestamp created_at = 3; // Creation timestamp
+ google.protobuf.Timestamp last_updated_at = 4; // Last update timestamp
+}
+
+/*
+ * Represents cached output, either as literals or an URI, with associated metadata.
+ */
+message CachedOutput {
+ oneof output {
+ flyteidl2.core.LiteralMap output_literals = 1; // Output literals
+ string output_uri = 2; // URI to output data
+ }
+ Metadata metadata = 3; // Associated metadata
+}
+
+/*
+ * Request to retrieve cached data by key.
+ */
+message GetCacheRequest {
+ string key = 1; // Cache key
+}
+
+/*
+ * Response with cached data for a given key.
+ */
+message GetCacheResponse {
+ CachedOutput output = 1; // Cached output
+}
+
+message OverwriteOutput {
+ bool overwrite = 1; // Overwrite flag
+ bool delete_blob = 2; // Delete existing blob
+ google.protobuf.Duration max_age = 3; // Maximum age of the cached output since last update
+}
+
+/*
+ * Request to store/update cached data by key.
+ */
+message PutCacheRequest {
+ string key = 1; // Cache key
+ CachedOutput output = 2; // Output to cache
+ OverwriteOutput overwrite = 3; // Overwrite flag if exists
+}
+
+/*
+ * Response message of cache store/update operation.
+ */
+message PutCacheResponse {
+ // Empty, success indicated by no errors
+}
+
+/*
+ * Request to delete cached data by key.
+ */
+message DeleteCacheRequest {
+ string key = 1; // Cache key
+}
+
+/*
+ * Response message of cache deletion operation.
+ */
+message DeleteCacheResponse {
+ // Empty, success indicated by no errors
+}
+
+// A reservation including owner, heartbeat interval, expiration timestamp, and various metadata.
+message Reservation {
+ string key = 1; // The unique ID for the reservation - same as the cache key
+ string owner_id = 2; // The unique ID of the owner for the reservation
+ google.protobuf.Duration heartbeat_interval = 3; // Requested reservation extension heartbeat interval
+ google.protobuf.Timestamp expires_at = 4; // Expiration timestamp of this reservation
+}
+
+/*
+ * Request to get or extend a reservation for a cache key
+ */
+message GetOrExtendReservationRequest {
+ string key = 1; // The unique ID for the reservation - same as the cache key
+ string owner_id = 2; // The unique ID of the owner for the reservation
+ google.protobuf.Duration heartbeat_interval = 3; // Requested reservation extension heartbeat interval
+}
+
+/*
+ * Request to get or extend a reservation for a cache key
+ */
+message GetOrExtendReservationResponse {
+ Reservation reservation = 1; // The reservation that was created or extended
+}
+
+/*
+ * Request to release the reservation for a cache key
+ */
+message ReleaseReservationRequest {
+ string key = 1; // The unique ID for the reservation - same as the cache key
+ string owner_id = 2; // The unique ID of the owner for the reservation
+}
+
+/*
+ * Response message of release reservation operation.
+ */
+message ReleaseReservationResponse {
+ // Empty, success indicated by no errors
+}
diff --git a/flyteidl2/cacheservice/v2/cacheservice.proto b/flyteidl2/cacheservice/v2/cacheservice.proto
new file mode 100644
index 00000000000..bf98b3e1ee5
--- /dev/null
+++ b/flyteidl2/cacheservice/v2/cacheservice.proto
@@ -0,0 +1,78 @@
+syntax = "proto3";
+
+package flyteidl2.cacheservice.v2;
+
+import "buf/validate/validate.proto";
+import "flyteidl2/cacheservice/cacheservice.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/cacheservice/v2";
+
+/*
+ * CacheService defines operations for cache management including retrieval, storage, and deletion of cached task/workflow outputs.
+ */
+service CacheService {
+ // Retrieves cached data by key.
+ rpc Get(GetCacheRequest) returns (flyteidl2.cacheservice.GetCacheResponse);
+
+ // Stores or updates cached data by key.
+ rpc Put(PutCacheRequest) returns (flyteidl2.cacheservice.PutCacheResponse);
+
+ // Deletes cached data by key.
+ rpc Delete(DeleteCacheRequest) returns (flyteidl2.cacheservice.DeleteCacheResponse);
+
+ // Get or extend a reservation for a cache key
+ rpc GetOrExtendReservation(GetOrExtendReservationRequest) returns (flyteidl2.cacheservice.GetOrExtendReservationResponse);
+
+ // Release the reservation for a cache key
+ rpc ReleaseReservation(ReleaseReservationRequest) returns (flyteidl2.cacheservice.ReleaseReservationResponse);
+}
+
+/*
+ * Identifier for cache operations, including org, project, and domain.
+ * This is used to scope cache operations to specific organizational contexts.
+ */
+message Identifier {
+ string org = 1 [(buf.validate.field).string.min_len = 1]; // Organization identifier
+ string project = 2 [(buf.validate.field).string.min_len = 1]; // Project identifier
+ string domain = 3 [(buf.validate.field).string.min_len = 1]; // Domain identifier
+}
+
+/*
+ * Request to retrieve cached data by key.
+ */
+message GetCacheRequest {
+ flyteidl2.cacheservice.GetCacheRequest base_request = 1;
+ Identifier identifier = 2 [(buf.validate.field).required = true]; // Identifier for the cache operation
+}
+
+/*
+ * Request to store/update cached data by key.
+ */
+message PutCacheRequest {
+ flyteidl2.cacheservice.PutCacheRequest base_request = 1;
+ Identifier identifier = 2 [(buf.validate.field).required = true]; // Identifier for the cache operation
+}
+
+/*
+ * Request to delete cached data by key.
+ */
+message DeleteCacheRequest {
+ flyteidl2.cacheservice.DeleteCacheRequest base_request = 1;
+ Identifier identifier = 2 [(buf.validate.field).required = true]; // Identifier for the cache operation
+}
+
+/*
+ * Request to get or extend a reservation for a cache key
+ */
+message GetOrExtendReservationRequest {
+ flyteidl2.cacheservice.GetOrExtendReservationRequest base_request = 1;
+ Identifier identifier = 2 [(buf.validate.field).required = true]; // Identifier for the cache operation
+}
+
+/*
+ * Request to release the reservation for a cache key
+ */
+message ReleaseReservationRequest {
+ flyteidl2.cacheservice.ReleaseReservationRequest base_request = 1;
+ Identifier identifier = 2 [(buf.validate.field).required = true]; // Identifier for the cache operation
+}
diff --git a/flyteidl2/clients/go/coreutils/extract_literal.go b/flyteidl2/clients/go/coreutils/extract_literal.go
new file mode 100644
index 00000000000..aa8fde2b916
--- /dev/null
+++ b/flyteidl2/clients/go/coreutils/extract_literal.go
@@ -0,0 +1,107 @@
+// extract_literal.go
+// Utility methods to extract a native golang value from a given Literal.
+// Usage:
+// 1] string literal extraction
+// lit, _ := MakeLiteral("test_string")
+// val, _ := ExtractFromLiteral(lit)
+// 2] integer literal extraction. integer would be extracted in type int64.
+// lit, _ := MakeLiteral([]interface{}{1, 2, 3})
+// val, _ := ExtractFromLiteral(lit)
+// 3] float literal extraction. float would be extracted in type float64.
+// lit, _ := MakeLiteral([]interface{}{1.0, 2.0, 3.0})
+// val, _ := ExtractFromLiteral(lit)
+// 4] map of boolean literal extraction.
+// mapInstance := map[string]interface{}{
+// "key1": []interface{}{1, 2, 3},
+// "key2": []interface{}{5},
+// }
+// lit, _ := MakeLiteral(mapInstance)
+// val, _ := ExtractFromLiteral(lit)
+// For further examples check the test TestFetchLiteral in extract_literal_test.go
+
+package coreutils
+
+import (
+ "fmt"
+
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+func ExtractFromLiteral(literal *core.Literal) (interface{}, error) {
+ switch literalValue := literal.Value.(type) {
+ case *core.Literal_Scalar:
+ switch scalarValue := literalValue.Scalar.Value.(type) {
+ case *core.Scalar_Primitive:
+ switch scalarPrimitive := scalarValue.Primitive.Value.(type) {
+ case *core.Primitive_Integer:
+ scalarPrimitiveInt := scalarPrimitive.Integer
+ return scalarPrimitiveInt, nil
+ case *core.Primitive_FloatValue:
+ scalarPrimitiveFloat := scalarPrimitive.FloatValue
+ return scalarPrimitiveFloat, nil
+ case *core.Primitive_StringValue:
+ scalarPrimitiveString := scalarPrimitive.StringValue
+ return scalarPrimitiveString, nil
+ case *core.Primitive_Boolean:
+ scalarPrimitiveBoolean := scalarPrimitive.Boolean
+ return scalarPrimitiveBoolean, nil
+ case *core.Primitive_Datetime:
+ scalarPrimitiveDateTime := scalarPrimitive.Datetime.AsTime()
+ return scalarPrimitiveDateTime, nil
+ case *core.Primitive_Duration:
+ scalarPrimitiveDuration := scalarPrimitive.Duration.AsDuration()
+ return scalarPrimitiveDuration, nil
+ default:
+ return nil, fmt.Errorf("unsupported literal scalar primitive type %T", scalarValue)
+ }
+ case *core.Scalar_Binary:
+ return scalarValue.Binary, nil
+ case *core.Scalar_Blob:
+ return scalarValue.Blob.Uri, nil
+ case *core.Scalar_Schema:
+ return scalarValue.Schema.Uri, nil
+ case *core.Scalar_Generic:
+ return scalarValue.Generic, nil
+ case *core.Scalar_StructuredDataset:
+ return scalarValue.StructuredDataset.Uri, nil
+ case *core.Scalar_Union:
+ // extract the value of the union but not the actual union object
+ extractedVal, err := ExtractFromLiteral(scalarValue.Union.Value)
+ if err != nil {
+ return nil, err
+ }
+ return extractedVal, nil
+ case *core.Scalar_NoneType:
+ return nil, nil
+ default:
+ return nil, fmt.Errorf("unsupported literal scalar type %T", scalarValue)
+ }
+ case *core.Literal_Collection:
+ collectionValue := literalValue.Collection.Literals
+ collection := make([]interface{}, len(collectionValue))
+ for index, val := range collectionValue {
+ if collectionElem, err := ExtractFromLiteral(val); err == nil {
+ collection[index] = collectionElem
+ } else {
+ return nil, err
+ }
+ }
+ return collection, nil
+ case *core.Literal_Map:
+ mapLiteralValue := literalValue.Map.Literals
+ mapResult := make(map[string]interface{}, len(mapLiteralValue))
+ for key, val := range mapLiteralValue {
+ if val, err := ExtractFromLiteral(val); err == nil {
+ mapResult[key] = val
+ } else {
+ return nil, err
+ }
+ }
+ return mapResult, nil
+ case *core.Literal_OffloadedMetadata:
+ // Return the URI of the offloaded metadata to be used when displaying in flytectl
+ return literalValue.OffloadedMetadata.Uri, nil
+
+ }
+ return nil, fmt.Errorf("unsupported literal type %T", literal)
+}
diff --git a/flyteidl2/clients/go/coreutils/extract_literal_test.go b/flyteidl2/clients/go/coreutils/extract_literal_test.go
new file mode 100644
index 00000000000..8781c6b3a52
--- /dev/null
+++ b/flyteidl2/clients/go/coreutils/extract_literal_test.go
@@ -0,0 +1,267 @@
+// extract_literal_test.go
+// Test class for the utility methods which extract a native golang value from a flyte Literal.
+
+package coreutils
+
+import (
+ "os"
+ "testing"
+ "time"
+
+ structpb "github.com/golang/protobuf/ptypes/struct"
+ "github.com/stretchr/testify/assert"
+
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+func TestFetchLiteral(t *testing.T) {
+ t.Run("Primitive", func(t *testing.T) {
+ lit, err := MakeLiteral("test_string")
+ assert.NoError(t, err)
+ val, err := ExtractFromLiteral(lit)
+ assert.NoError(t, err)
+ assert.Equal(t, "test_string", val)
+ })
+
+ t.Run("Timestamp", func(t *testing.T) {
+ now := time.Now().UTC()
+ lit, err := MakeLiteral(now)
+ assert.NoError(t, err)
+ val, err := ExtractFromLiteral(lit)
+ assert.NoError(t, err)
+ assert.Equal(t, now, val)
+ })
+
+ t.Run("Duration", func(t *testing.T) {
+ duration := time.Second * 10
+ lit, err := MakeLiteral(duration)
+ assert.NoError(t, err)
+ val, err := ExtractFromLiteral(lit)
+ assert.NoError(t, err)
+ assert.Equal(t, duration, val)
+ })
+
+ t.Run("Array", func(t *testing.T) {
+ lit, err := MakeLiteral([]interface{}{1, 2, 3})
+ assert.NoError(t, err)
+ val, err := ExtractFromLiteral(lit)
+ assert.NoError(t, err)
+ arr := []interface{}{int64(1), int64(2), int64(3)}
+ assert.Equal(t, arr, val)
+ })
+
+ t.Run("Map", func(t *testing.T) {
+ mapInstance := map[string]interface{}{
+ "key1": []interface{}{1, 2, 3},
+ "key2": []interface{}{5},
+ }
+ lit, err := MakeLiteral(mapInstance)
+ assert.NoError(t, err)
+ val, err := ExtractFromLiteral(lit)
+ assert.NoError(t, err)
+ expectedMapInstance := map[string]interface{}{
+ "key1": []interface{}{int64(1), int64(2), int64(3)},
+ "key2": []interface{}{int64(5)},
+ }
+ assert.Equal(t, expectedMapInstance, val)
+ })
+
+ t.Run("Map_Booleans", func(t *testing.T) {
+ mapInstance := map[string]interface{}{
+ "key1": []interface{}{true, false, true},
+ "key2": []interface{}{false},
+ }
+ lit, err := MakeLiteral(mapInstance)
+ assert.NoError(t, err)
+ val, err := ExtractFromLiteral(lit)
+ assert.NoError(t, err)
+ assert.Equal(t, mapInstance, val)
+ })
+
+ t.Run("Map_Floats", func(t *testing.T) {
+ mapInstance := map[string]interface{}{
+ "key1": []interface{}{1.0, 2.0, 3.0},
+ "key2": []interface{}{1.0},
+ }
+ lit, err := MakeLiteral(mapInstance)
+ assert.NoError(t, err)
+ val, err := ExtractFromLiteral(lit)
+ assert.NoError(t, err)
+ expectedMapInstance := map[string]interface{}{
+ "key1": []interface{}{float64(1.0), float64(2.0), float64(3.0)},
+ "key2": []interface{}{float64(1.0)},
+ }
+ assert.Equal(t, expectedMapInstance, val)
+ })
+
+ t.Run("NestedMap", func(t *testing.T) {
+ mapInstance := map[string]interface{}{
+ "key1": map[string]interface{}{"key11": 1.0, "key12": 2.0, "key13": 3.0},
+ "key2": map[string]interface{}{"key21": 1.0},
+ }
+ lit, err := MakeLiteral(mapInstance)
+ assert.NoError(t, err)
+ val, err := ExtractFromLiteral(lit)
+ assert.NoError(t, err)
+ expectedMapInstance := map[string]interface{}{
+ "key1": map[string]interface{}{"key11": float64(1.0), "key12": float64(2.0), "key13": float64(3.0)},
+ "key2": map[string]interface{}{"key21": float64(1.0)},
+ }
+ assert.Equal(t, expectedMapInstance, val)
+ })
+
+ t.Run("Binary", func(t *testing.T) {
+ s := MakeBinaryLiteral([]byte{'h'})
+ assert.Equal(t, []byte{'h'}, s.GetScalar().GetBinary().GetValue())
+ _, err := ExtractFromLiteral(s)
+ assert.Nil(t, err)
+ })
+
+ t.Run("NoneType", func(t *testing.T) {
+ p, err := MakeLiteral(nil)
+ assert.NoError(t, err)
+ assert.NotNil(t, p.GetScalar())
+ _, err = ExtractFromLiteral(p)
+ assert.Nil(t, err)
+ })
+
+ t.Run("Generic", func(t *testing.T) {
+ os.Setenv(FlyteUseOldDcFormat, "true")
+ literalVal := map[string]interface{}{
+ "x": 1,
+ "y": "ystringvalue",
+ }
+ var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRUCT}}
+ lit, err := MakeLiteralForType(literalType, literalVal)
+ assert.NoError(t, err)
+ extractedLiteralVal, err := ExtractFromLiteral(lit)
+ assert.NoError(t, err)
+ fieldsMap := map[string]*structpb.Value{
+ "x": {
+ Kind: &structpb.Value_NumberValue{NumberValue: 1},
+ },
+ "y": {
+ Kind: &structpb.Value_StringValue{StringValue: "ystringvalue"},
+ },
+ }
+ expectedStructVal := &structpb.Struct{
+ Fields: fieldsMap,
+ }
+ extractedStructValue := extractedLiteralVal.(*structpb.Struct)
+ assert.Equal(t, len(expectedStructVal.Fields), len(extractedStructValue.Fields))
+ for key, val := range expectedStructVal.Fields {
+ assert.Equal(t, val.Kind, extractedStructValue.Fields[key].Kind)
+ }
+ os.Unsetenv(FlyteUseOldDcFormat)
+ })
+
+ t.Run("Generic Passed As String", func(t *testing.T) {
+ literalVal := "{\"x\": 1,\"y\": \"ystringvalue\"}"
+ var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRUCT}}
+ lit, err := MakeLiteralForType(literalType, literalVal)
+ assert.NoError(t, err)
+ extractedLiteralVal, err := ExtractFromLiteral(lit)
+ assert.NoError(t, err)
+ fieldsMap := map[string]*structpb.Value{
+ "x": {
+ Kind: &structpb.Value_NumberValue{NumberValue: 1},
+ },
+ "y": {
+ Kind: &structpb.Value_StringValue{StringValue: "ystringvalue"},
+ },
+ }
+ expectedStructVal := &structpb.Struct{
+ Fields: fieldsMap,
+ }
+ extractedStructValue := extractedLiteralVal.(*structpb.Struct)
+ assert.Equal(t, len(expectedStructVal.Fields), len(extractedStructValue.Fields))
+ for key, val := range expectedStructVal.Fields {
+ assert.Equal(t, val.Kind, extractedStructValue.Fields[key].Kind)
+ }
+ })
+
+ t.Run("Structured dataset", func(t *testing.T) {
+ literalVal := "s3://blah/blah/blah"
+ var dataSetColumns []*core.StructuredDatasetType_DatasetColumn
+ dataSetColumns = append(dataSetColumns, &core.StructuredDatasetType_DatasetColumn{
+ Name: "Price",
+ LiteralType: &core.LiteralType{
+ Type: &core.LiteralType_Simple{
+ Simple: core.SimpleType_FLOAT,
+ },
+ },
+ })
+ var literalType = &core.LiteralType{Type: &core.LiteralType_StructuredDatasetType{StructuredDatasetType: &core.StructuredDatasetType{
+ Columns: dataSetColumns,
+ Format: "testFormat",
+ }}}
+
+ lit, err := MakeLiteralForType(literalType, literalVal)
+ assert.NoError(t, err)
+ extractedLiteralVal, err := ExtractFromLiteral(lit)
+ assert.NoError(t, err)
+ assert.Equal(t, literalVal, extractedLiteralVal)
+ })
+
+ t.Run("Offloaded metadata", func(t *testing.T) {
+ literalVal := "s3://blah/blah/blah"
+ var storedLiteralType = &core.LiteralType{
+ Type: &core.LiteralType_CollectionType{
+ CollectionType: &core.LiteralType{
+ Type: &core.LiteralType_Simple{
+ Simple: core.SimpleType_INTEGER,
+ },
+ },
+ },
+ }
+ offloadedLiteral := &core.Literal{
+ Value: &core.Literal_OffloadedMetadata{
+ OffloadedMetadata: &core.LiteralOffloadedMetadata{
+ Uri: literalVal,
+ InferredType: storedLiteralType,
+ },
+ },
+ }
+ extractedLiteralVal, err := ExtractFromLiteral(offloadedLiteral)
+ assert.NoError(t, err)
+ assert.Equal(t, literalVal, extractedLiteralVal)
+ })
+
+ t.Run("Union", func(t *testing.T) {
+ literalVal := int64(1)
+ var literalType = &core.LiteralType{
+ Type: &core.LiteralType_UnionType{
+ UnionType: &core.UnionType{
+ Variants: []*core.LiteralType{
+ {Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}},
+ {Type: &core.LiteralType_Simple{Simple: core.SimpleType_FLOAT}},
+ },
+ },
+ },
+ }
+ lit, err := MakeLiteralForType(literalType, literalVal)
+ assert.NoError(t, err)
+ extractedLiteralVal, err := ExtractFromLiteral(lit)
+ assert.NoError(t, err)
+ assert.Equal(t, literalVal, extractedLiteralVal)
+ })
+
+ t.Run("Union with None", func(t *testing.T) {
+ var literalType = &core.LiteralType{
+ Type: &core.LiteralType_UnionType{
+ UnionType: &core.UnionType{
+ Variants: []*core.LiteralType{
+ {Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}},
+ {Type: &core.LiteralType_Simple{Simple: core.SimpleType_NONE}},
+ },
+ },
+ },
+ }
+ lit, err := MakeLiteralForType(literalType, nil)
+
+ assert.NoError(t, err)
+ extractedLiteralVal, err := ExtractFromLiteral(lit)
+ assert.NoError(t, err)
+ assert.Nil(t, extractedLiteralVal)
+ })
+}
diff --git a/flyteidl2/clients/go/coreutils/literals.go b/flyteidl2/clients/go/coreutils/literals.go
new file mode 100644
index 00000000000..8a1f882a64b
--- /dev/null
+++ b/flyteidl2/clients/go/coreutils/literals.go
@@ -0,0 +1,670 @@
+// Contains convenience methods for constructing core types.
+package coreutils
+
+import (
+ "encoding/json"
+ "fmt"
+ "math"
+ "os"
+ "reflect"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/golang/protobuf/jsonpb"
+ "github.com/golang/protobuf/ptypes"
+ structpb "github.com/golang/protobuf/ptypes/struct"
+ "github.com/pkg/errors"
+ "github.com/shamaton/msgpack/v2"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+const MESSAGEPACK = "msgpack"
+const FlyteUseOldDcFormat = "FLYTE_USE_OLD_DC_FORMAT"
+
+func MakePrimitive(v interface{}) (*core.Primitive, error) {
+ switch p := v.(type) {
+ case int:
+ return &core.Primitive{
+ Value: &core.Primitive_Integer{
+ Integer: int64(p),
+ },
+ }, nil
+ case int64:
+ return &core.Primitive{
+ Value: &core.Primitive_Integer{
+ Integer: p,
+ },
+ }, nil
+ case float64:
+ return &core.Primitive{
+ Value: &core.Primitive_FloatValue{
+ FloatValue: p,
+ },
+ }, nil
+ case time.Time:
+ t, err := ptypes.TimestampProto(p)
+ if err != nil {
+ return nil, err
+ }
+ return &core.Primitive{
+ Value: &core.Primitive_Datetime{
+ Datetime: t,
+ },
+ }, nil
+ case time.Duration:
+ d := ptypes.DurationProto(p)
+ return &core.Primitive{
+ Value: &core.Primitive_Duration{
+ Duration: d,
+ },
+ }, nil
+ case string:
+ return &core.Primitive{
+ Value: &core.Primitive_StringValue{
+ StringValue: p,
+ },
+ }, nil
+ case bool:
+ return &core.Primitive{
+ Value: &core.Primitive_Boolean{
+ Boolean: p,
+ },
+ }, nil
+ }
+ return nil, fmt.Errorf("failed to convert to a known primitive type. Input Type [%v] not supported", reflect.TypeOf(v).String())
+}
+
+func MustMakePrimitive(v interface{}) *core.Primitive {
+ f, err := MakePrimitive(v)
+ if err != nil {
+ panic(err)
+ }
+ return f
+}
+
+func MakePrimitiveLiteral(v interface{}) (*core.Literal, error) {
+ p, err := MakePrimitive(v)
+ if err != nil {
+ return nil, err
+ }
+ return &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{
+ Primitive: p,
+ },
+ },
+ },
+ }, nil
+}
+
+func MustMakePrimitiveLiteral(v interface{}) *core.Literal {
+ p, err := MakePrimitiveLiteral(v)
+ if err != nil {
+ panic(err)
+ }
+ return p
+}
+
+func MakeLiteralForMap(v map[string]interface{}) (*core.Literal, error) {
+ m, err := MakeLiteralMap(v)
+ if err != nil {
+ return nil, err
+ }
+
+ return &core.Literal{
+ Value: &core.Literal_Map{
+ Map: m,
+ },
+ }, nil
+}
+
+func MakeLiteralForCollection(v []interface{}) (*core.Literal, error) {
+ literals := make([]*core.Literal, 0, len(v))
+ for _, val := range v {
+ l, err := MakeLiteral(val)
+ if err != nil {
+ return nil, err
+ }
+
+ literals = append(literals, l)
+ }
+
+ return &core.Literal{
+ Value: &core.Literal_Collection{
+ Collection: &core.LiteralCollection{
+ Literals: literals,
+ },
+ },
+ }, nil
+}
+
+func MakeBinaryLiteral(v []byte) *core.Literal {
+ return &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Binary{
+ Binary: &core.Binary{
+ Value: v,
+ Tag: MESSAGEPACK,
+ },
+ },
+ },
+ },
+ }
+}
+
+func MakeGenericLiteral(v *structpb.Struct) *core.Literal {
+ return &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Generic{
+ Generic: v,
+ },
+ },
+ }}
+}
+
+func MakeLiteral(v interface{}) (*core.Literal, error) {
+ if v == nil {
+ return &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_NoneType{
+ NoneType: &core.Void{},
+ },
+ },
+ },
+ }, nil
+ }
+ switch o := v.(type) {
+ case *core.Literal:
+ return o, nil
+ case []interface{}:
+ return MakeLiteralForCollection(o)
+ case map[string]interface{}:
+ return MakeLiteralForMap(o)
+ case []byte:
+ return MakeBinaryLiteral(v.([]byte)), nil
+ case *structpb.Struct:
+ return MakeGenericLiteral(v.(*structpb.Struct)), nil
+ case *core.Error:
+ return &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Error{
+ Error: v.(*core.Error),
+ },
+ },
+ },
+ }, nil
+ default:
+ return MakePrimitiveLiteral(o)
+ }
+}
+
+func MustMakeDefaultLiteralForType(typ *core.LiteralType) *core.Literal {
+ if res, err := MakeDefaultLiteralForType(typ); err != nil {
+ panic(err)
+ } else {
+ return res
+ }
+}
+
+func MakeDefaultLiteralForType(typ *core.LiteralType) (*core.Literal, error) {
+ switch t := typ.GetType().(type) {
+ case *core.LiteralType_Simple:
+ switch t.Simple {
+ case core.SimpleType_NONE:
+ return MakeLiteral(nil)
+ case core.SimpleType_INTEGER:
+ return MakeLiteral(int(0))
+ case core.SimpleType_FLOAT:
+ return MakeLiteral(float64(0))
+ case core.SimpleType_STRING:
+ return MakeLiteral("")
+ case core.SimpleType_BOOLEAN:
+ return MakeLiteral(false)
+ case core.SimpleType_DATETIME:
+ return MakeLiteral(time.Now())
+ case core.SimpleType_DURATION:
+ return MakeLiteral(time.Second)
+ case core.SimpleType_BINARY:
+ return MakeLiteral([]byte{})
+ case core.SimpleType_ERROR:
+ return &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Error{
+ Error: &core.Error{
+ Message: "Default Error message",
+ },
+ },
+ },
+ },
+ }, nil
+ case core.SimpleType_STRUCT:
+ return &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Generic{
+ Generic: &structpb.Struct{},
+ },
+ },
+ },
+ }, nil
+ }
+ return nil, errors.Errorf("Not yet implemented. Default creation is not yet implemented for [%s] ", t.Simple.String())
+ case *core.LiteralType_Blob:
+ return &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Blob{
+ Blob: &core.Blob{
+ Metadata: &core.BlobMetadata{
+ Type: t.Blob,
+ },
+ Uri: "/tmp/somepath",
+ },
+ },
+ },
+ },
+ }, nil
+ case *core.LiteralType_CollectionType:
+ single, err := MakeDefaultLiteralForType(t.CollectionType)
+ if err != nil {
+ return nil, err
+ }
+
+ return &core.Literal{
+ Value: &core.Literal_Collection{
+ Collection: &core.LiteralCollection{
+ Literals: []*core.Literal{single},
+ },
+ },
+ }, nil
+ case *core.LiteralType_MapValueType:
+ single, err := MakeDefaultLiteralForType(t.MapValueType)
+ if err != nil {
+ return nil, err
+ }
+
+ return &core.Literal{
+ Value: &core.Literal_Map{
+ Map: &core.LiteralMap{
+ Literals: map[string]*core.Literal{
+ "itemKey": single,
+ },
+ },
+ },
+ }, nil
+ case *core.LiteralType_EnumType:
+ return MakeLiteralForType(typ, nil)
+ case *core.LiteralType_Schema:
+ return MakeLiteralForType(typ, nil)
+ case *core.LiteralType_UnionType:
+ if len(t.UnionType.Variants) == 0 {
+ return nil, errors.Errorf("Union type must have at least one variant")
+ }
+ // For union types, we just return the default for the first variant
+ val, err := MakeDefaultLiteralForType(t.UnionType.Variants[0])
+ if err != nil {
+ return nil, errors.Errorf("Failed to create default literal for first union type variant [%v]", t.UnionType.Variants[0])
+ }
+ res := &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Union{
+ Union: &core.Union{
+ Type: t.UnionType.Variants[0],
+ Value: val,
+ },
+ },
+ },
+ },
+ }
+ return res, nil
+ }
+
+ return nil, fmt.Errorf("failed to convert to a known Literal. Input Type [%v] not supported", typ.String())
+}
+
+func MakePrimitiveForType(t core.SimpleType, s string) (*core.Primitive, error) {
+ p := &core.Primitive{}
+ switch t {
+ case core.SimpleType_INTEGER:
+ v, err := strconv.ParseInt(s, 10, 64)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to parse integer value")
+ }
+ p.Value = &core.Primitive_Integer{Integer: v}
+ case core.SimpleType_FLOAT:
+ v, err := strconv.ParseFloat(s, 64)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to parse Float value")
+ }
+ p.Value = &core.Primitive_FloatValue{FloatValue: v}
+ case core.SimpleType_BOOLEAN:
+ v, err := strconv.ParseBool(s)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to parse Bool value")
+ }
+ p.Value = &core.Primitive_Boolean{Boolean: v}
+ case core.SimpleType_STRING:
+ p.Value = &core.Primitive_StringValue{StringValue: s}
+ case core.SimpleType_DURATION:
+ v, err := time.ParseDuration(s)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to parse Duration, valid formats: e.g. 300ms, -1.5h, 2h45m")
+ }
+ p.Value = &core.Primitive_Duration{Duration: ptypes.DurationProto(v)}
+ case core.SimpleType_DATETIME:
+ v, err := time.Parse(time.RFC3339, s)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to parse Datetime in RFC3339 format")
+ }
+ ts, err := ptypes.TimestampProto(v)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to convert datetime to proto")
+ }
+ p.Value = &core.Primitive_Datetime{Datetime: ts}
+ default:
+ return nil, fmt.Errorf("unsupported type %s", t.String())
+ }
+ return p, nil
+}
+
+func MakeLiteralForSimpleType(t core.SimpleType, s string) (*core.Literal, error) {
+ s = strings.Trim(s, " \n\t")
+ scalar := &core.Scalar{}
+ switch t {
+ case core.SimpleType_STRUCT:
+ st := &structpb.Struct{}
+ unmarshaler := jsonpb.Unmarshaler{AllowUnknownFields: true}
+ err := unmarshaler.Unmarshal(strings.NewReader(s), st)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to load generic type as json.")
+ }
+ scalar.Value = &core.Scalar_Generic{
+ Generic: st,
+ }
+ case core.SimpleType_BINARY:
+ scalar.Value = &core.Scalar_Binary{
+ Binary: &core.Binary{
+ Value: []byte(s),
+ Tag: MESSAGEPACK,
+ },
+ }
+ case core.SimpleType_ERROR:
+ scalar.Value = &core.Scalar_Error{
+ Error: &core.Error{
+ Message: s,
+ },
+ }
+ case core.SimpleType_NONE:
+ scalar.Value = &core.Scalar_NoneType{
+ NoneType: &core.Void{},
+ }
+ default:
+ p, err := MakePrimitiveForType(t, s)
+ if err != nil {
+ return nil, err
+ }
+ scalar.Value = &core.Scalar_Primitive{Primitive: p}
+ }
+ return &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: scalar,
+ },
+ }, nil
+}
+
+func MustMakeLiteral(v interface{}) *core.Literal {
+ p, err := MakeLiteral(v)
+ if err != nil {
+ panic(err)
+ }
+
+ return p
+}
+
+func MakeLiteralMap(v map[string]interface{}) (*core.LiteralMap, error) {
+
+ literals := make(map[string]*core.Literal, len(v))
+ for key, val := range v {
+ l, err := MakeLiteral(val)
+ if err != nil {
+ return nil, err
+ }
+
+ literals[key] = l
+ }
+
+ return &core.LiteralMap{
+ Literals: literals,
+ }, nil
+}
+
+func MakeLiteralForSchema(path storage.DataReference, columns []*core.SchemaType_SchemaColumn) *core.Literal {
+ return &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Schema{
+ Schema: &core.Schema{
+ Uri: path.String(),
+ Type: &core.SchemaType{
+ Columns: columns,
+ },
+ },
+ },
+ },
+ },
+ }
+}
+
+func MakeLiteralForStructuredDataSet(path storage.DataReference, columns []*core.StructuredDatasetType_DatasetColumn, format string) *core.Literal {
+ return &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_StructuredDataset{
+ StructuredDataset: &core.StructuredDataset{
+ Uri: path.String(),
+ Metadata: &core.StructuredDatasetMetadata{
+ StructuredDatasetType: &core.StructuredDatasetType{
+ Columns: columns,
+ Format: format,
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+}
+
+func MakeLiteralForBlob(path storage.DataReference, isDir bool, format string) *core.Literal {
+ dim := core.BlobType_SINGLE
+ if isDir {
+ dim = core.BlobType_MULTIPART
+ }
+ return &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Blob{
+ Blob: &core.Blob{
+ Uri: path.String(),
+ Metadata: &core.BlobMetadata{
+ Type: &core.BlobType{
+ Dimensionality: dim,
+ Format: format,
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+}
+
+func MakeLiteralForType(t *core.LiteralType, v interface{}) (*core.Literal, error) {
+ l := &core.Literal{}
+ switch newT := t.Type.(type) {
+ case *core.LiteralType_MapValueType:
+ newV, ok := v.(map[string]interface{})
+ if !ok {
+ return nil, fmt.Errorf("map value types can only be of type map[string]interface{}, but found %v", reflect.TypeOf(v))
+ }
+
+ literals := make(map[string]*core.Literal, len(newV))
+ for key, val := range newV {
+ lv, err := MakeLiteralForType(newT.MapValueType, val)
+ if err != nil {
+ return nil, err
+ }
+ literals[key] = lv
+ }
+ l.Value = &core.Literal_Map{
+ Map: &core.LiteralMap{
+ Literals: literals,
+ },
+ }
+
+ case *core.LiteralType_CollectionType:
+ newV, ok := v.([]interface{})
+ if !ok {
+ return nil, fmt.Errorf("collection type expected but found %v", reflect.TypeOf(v))
+ }
+
+ literals := make([]*core.Literal, 0, len(newV))
+ for _, val := range newV {
+ lv, err := MakeLiteralForType(newT.CollectionType, val)
+ if err != nil {
+ return nil, err
+ }
+ literals = append(literals, lv)
+ }
+ l.Value = &core.Literal_Collection{
+ Collection: &core.LiteralCollection{
+ Literals: literals,
+ },
+ }
+
+ case *core.LiteralType_Simple:
+ strValue := fmt.Sprintf("%v", v)
+ if v == nil {
+ strValue = ""
+ }
+ // Note this is to support large integers which by default when passed from an unmarshalled json will be
+ // converted to float64 and printed as exponential format by Sprintf.
+ // eg : 8888888 get converted to 8.888888e+06 and which causes strconv.ParseInt to fail
+ // Inorder to avoid this we explicitly add this check.
+ if f, ok := v.(float64); ok && math.Trunc(f) == f {
+ strValue = fmt.Sprintf("%.0f", math.Trunc(f))
+ }
+ if newT.Simple == core.SimpleType_STRUCT {
+ useOldFormat := strings.ToLower(os.Getenv(FlyteUseOldDcFormat))
+ if _, isValueStringType := v.(string); !isValueStringType {
+ if useOldFormat == "1" || useOldFormat == "t" || useOldFormat == "true" {
+ byteValue, err := json.Marshal(v)
+ if err != nil {
+ return nil, fmt.Errorf("unable to marshal to json string for struct value %v", v)
+ }
+ strValue = string(byteValue)
+ } else {
+ byteValue, err := msgpack.Marshal(v)
+ if err != nil {
+ return nil, fmt.Errorf("unable to marshal to msgpack bytes for struct value %v", v)
+ }
+ return &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Binary{
+ Binary: &core.Binary{
+ Value: byteValue,
+ Tag: MESSAGEPACK,
+ },
+ },
+ },
+ },
+ }, nil
+ }
+ }
+ }
+ lv, err := MakeLiteralForSimpleType(newT.Simple, strValue)
+ if err != nil {
+ return nil, err
+ }
+ return lv, nil
+
+ case *core.LiteralType_Blob:
+ isDir := newT.Blob.Dimensionality == core.BlobType_MULTIPART
+ lv := MakeLiteralForBlob(storage.DataReference(fmt.Sprintf("%v", v)), isDir, newT.Blob.Format)
+ return lv, nil
+
+ case *core.LiteralType_Schema:
+ lv := MakeLiteralForSchema(storage.DataReference(fmt.Sprintf("%v", v)), newT.Schema.Columns)
+ return lv, nil
+ case *core.LiteralType_StructuredDatasetType:
+ lv := MakeLiteralForStructuredDataSet(storage.DataReference(fmt.Sprintf("%v", v)), newT.StructuredDatasetType.Columns, newT.StructuredDatasetType.Format)
+ return lv, nil
+
+ case *core.LiteralType_EnumType:
+ var newV string
+ if v == nil {
+ if len(t.GetEnumType().Values) == 0 {
+ return nil, fmt.Errorf("enum types need at least one value")
+ }
+ newV = t.GetEnumType().Values[0]
+ } else {
+ var ok bool
+ newV, ok = v.(string)
+ if !ok {
+ return nil, fmt.Errorf("cannot convert [%v] to enum representations, only string values are supported in enum literals", reflect.TypeOf(v))
+ }
+ found := false
+ for _, val := range t.GetEnumType().GetValues() {
+ if val == newV {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return nil, fmt.Errorf("incorrect enum value [%s], supported values %+v", newV, t.GetEnumType().GetValues())
+ }
+ }
+ return MakePrimitiveLiteral(newV)
+
+ case *core.LiteralType_UnionType:
+ // Try different types in the variants, return the first one matched
+ found := false
+ for _, subType := range newT.UnionType.Variants {
+ lv, err := MakeLiteralForType(subType, v)
+ if err == nil {
+ l = &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Union{
+ Union: &core.Union{
+ Value: lv,
+ Type: subType,
+ },
+ },
+ },
+ },
+ }
+ found = true
+ break
+ }
+ }
+ if !found {
+ return nil, fmt.Errorf("incorrect union value [%s], supported values %+v", v, newT.UnionType.Variants)
+ }
+ default:
+ return nil, fmt.Errorf("unsupported type %s", t.String())
+ }
+
+ return l, nil
+}
diff --git a/flyteidl2/clients/go/coreutils/literals_test.go b/flyteidl2/clients/go/coreutils/literals_test.go
new file mode 100644
index 00000000000..fecb19840c3
--- /dev/null
+++ b/flyteidl2/clients/go/coreutils/literals_test.go
@@ -0,0 +1,868 @@
+// extract_literal_test.go
+// Test class for the utility methods which construct flyte literals.
+
+package coreutils
+
+import (
+ "fmt"
+ "os"
+ "reflect"
+ "strconv"
+ "testing"
+ "time"
+
+ "github.com/flyteorg/flyte/v2/flytestdlib/storage"
+ "github.com/go-test/deep"
+ "github.com/golang/protobuf/ptypes"
+ structpb "github.com/golang/protobuf/ptypes/struct"
+ "github.com/pkg/errors"
+ "github.com/shamaton/msgpack/v2"
+ "github.com/stretchr/testify/assert"
+
+ "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core"
+)
+
+func TestMakePrimitive(t *testing.T) {
+ {
+ v := 1
+ p, err := MakePrimitive(v)
+ assert.NoError(t, err)
+ assert.Equal(t, "*core.Primitive_Integer", reflect.TypeOf(p.Value).String())
+ assert.Equal(t, int64(v), p.GetInteger())
+ }
+ {
+ v := int64(1)
+ p, err := MakePrimitive(v)
+ assert.NoError(t, err)
+ assert.Equal(t, "*core.Primitive_Integer", reflect.TypeOf(p.Value).String())
+ assert.Equal(t, v, p.GetInteger())
+ }
+ {
+ v := 1.0
+ p, err := MakePrimitive(v)
+ assert.NoError(t, err)
+ assert.Equal(t, "*core.Primitive_FloatValue", reflect.TypeOf(p.Value).String())
+ assert.Equal(t, v, p.GetFloatValue())
+ }
+ {
+ v := "blah"
+ p, err := MakePrimitive(v)
+ assert.NoError(t, err)
+ assert.Equal(t, "*core.Primitive_StringValue", reflect.TypeOf(p.Value).String())
+ assert.Equal(t, v, p.GetStringValue())
+ }
+ {
+ v := true
+ p, err := MakePrimitive(v)
+ assert.NoError(t, err)
+ assert.Equal(t, "*core.Primitive_Boolean", reflect.TypeOf(p.Value).String())
+ assert.Equal(t, v, p.GetBoolean())
+ }
+ {
+ v := time.Now()
+ p, err := MakePrimitive(v)
+ assert.NoError(t, err)
+ assert.Equal(t, "*core.Primitive_Datetime", reflect.TypeOf(p.Value).String())
+ j, err := ptypes.TimestampProto(v)
+ assert.NoError(t, err)
+ assert.Equal(t, j, p.GetDatetime())
+ _, err = MakePrimitive(time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC))
+ assert.Error(t, err)
+ }
+ {
+ v := time.Second * 10
+ p, err := MakePrimitive(v)
+ assert.NoError(t, err)
+ assert.Equal(t, "*core.Primitive_Duration", reflect.TypeOf(p.Value).String())
+ assert.Equal(t, ptypes.DurationProto(v), p.GetDuration())
+ }
+ {
+ v := struct {
+ }{}
+ _, err := MakePrimitive(v)
+ assert.Error(t, err)
+ }
+}
+
+func TestMustMakePrimitive(t *testing.T) {
+ {
+ v := struct {
+ }{}
+ assert.Panics(t, func() {
+ MustMakePrimitive(v)
+ })
+ }
+ {
+ v := time.Second * 10
+ p := MustMakePrimitive(v)
+ assert.Equal(t, "*core.Primitive_Duration", reflect.TypeOf(p.Value).String())
+ assert.Equal(t, ptypes.DurationProto(v), p.GetDuration())
+ }
+}
+
+func TestMakePrimitiveLiteral(t *testing.T) {
+ {
+ v := 1.0
+ p, err := MakePrimitiveLiteral(v)
+ assert.NoError(t, err)
+ assert.NotNil(t, p.GetScalar())
+ assert.Equal(t, "*core.Primitive_FloatValue", reflect.TypeOf(p.GetScalar().GetPrimitive().Value).String())
+ assert.Equal(t, v, p.GetScalar().GetPrimitive().GetFloatValue())
+ }
+ {
+ v := struct {
+ }{}
+ _, err := MakePrimitiveLiteral(v)
+ assert.Error(t, err)
+ }
+}
+
+func TestMustMakePrimitiveLiteral(t *testing.T) {
+ t.Run("Panic", func(t *testing.T) {
+ v := struct {
+ }{}
+ assert.Panics(t, func() {
+ MustMakePrimitiveLiteral(v)
+ })
+ })
+ t.Run("FloatValue", func(t *testing.T) {
+ v := 1.0
+ p := MustMakePrimitiveLiteral(v)
+ assert.NotNil(t, p.GetScalar())
+ assert.Equal(t, "*core.Primitive_FloatValue", reflect.TypeOf(p.GetScalar().GetPrimitive().Value).String())
+ assert.Equal(t, v, p.GetScalar().GetPrimitive().GetFloatValue())
+ })
+}
+
+func TestMakeLiteral(t *testing.T) {
+ t.Run("Primitive", func(t *testing.T) {
+ lit, err := MakeLiteral("test_string")
+ assert.NoError(t, err)
+ assert.Equal(t, "*core.Primitive_StringValue", reflect.TypeOf(lit.GetScalar().GetPrimitive().Value).String())
+ })
+
+ t.Run("Array", func(t *testing.T) {
+ lit, err := MakeLiteral([]interface{}{1, 2, 3})
+ assert.NoError(t, err)
+ assert.Equal(t, "*core.Literal_Collection", reflect.TypeOf(lit.GetValue()).String())
+ assert.Equal(t, "*core.Primitive_Integer", reflect.TypeOf(lit.GetCollection().Literals[0].GetScalar().GetPrimitive().Value).String())
+ })
+
+ t.Run("Map", func(t *testing.T) {
+ lit, err := MakeLiteral(map[string]interface{}{
+ "key1": []interface{}{1, 2, 3},
+ "key2": []interface{}{5},
+ })
+ assert.NoError(t, err)
+ assert.Equal(t, "*core.Literal_Map", reflect.TypeOf(lit.GetValue()).String())
+ assert.Equal(t, "*core.Literal_Collection", reflect.TypeOf(lit.GetMap().Literals["key1"].GetValue()).String())
+ })
+
+ t.Run("Binary", func(t *testing.T) {
+ s := MakeBinaryLiteral([]byte{'h'})
+ assert.Equal(t, []byte{'h'}, s.GetScalar().GetBinary().GetValue())
+ })
+
+ t.Run("NoneType", func(t *testing.T) {
+ p, err := MakeLiteral(nil)
+ assert.NoError(t, err)
+ assert.NotNil(t, p.GetScalar())
+ assert.Equal(t, "*core.Scalar_NoneType", reflect.TypeOf(p.GetScalar().Value).String())
+ })
+}
+
+func TestMustMakeLiteral(t *testing.T) {
+ v := "hello"
+ l := MustMakeLiteral(v)
+ assert.NotNil(t, l.GetScalar())
+ assert.Equal(t, v, l.GetScalar().GetPrimitive().GetStringValue())
+}
+
+func TestMakeBinaryLiteral(t *testing.T) {
+ s := MakeBinaryLiteral([]byte{'h'})
+ assert.Equal(t, []byte{'h'}, s.GetScalar().GetBinary().GetValue())
+}
+
+func TestMakeDefaultLiteralForType(t *testing.T) {
+ type args struct {
+ name string
+ ty core.SimpleType
+ tyName string
+ isPrimitive bool
+ }
+ tests := []args{
+ {"None", core.SimpleType_NONE, "*core.Scalar_NoneType", false},
+ {"Binary", core.SimpleType_BINARY, "*core.Scalar_Binary", false},
+ {"Integer", core.SimpleType_INTEGER, "*core.Primitive_Integer", true},
+ {"Float", core.SimpleType_FLOAT, "*core.Primitive_FloatValue", true},
+ {"String", core.SimpleType_STRING, "*core.Primitive_StringValue", true},
+ {"Boolean", core.SimpleType_BOOLEAN, "*core.Primitive_Boolean", true},
+ {"Duration", core.SimpleType_DURATION, "*core.Primitive_Duration", true},
+ {"Datetime", core.SimpleType_DATETIME, "*core.Primitive_Datetime", true},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Simple{Simple: test.ty}})
+ assert.NoError(t, err)
+ if test.isPrimitive {
+ assert.Equal(t, test.tyName, reflect.TypeOf(l.GetScalar().GetPrimitive().Value).String())
+ } else {
+ assert.Equal(t, test.tyName, reflect.TypeOf(l.GetScalar().Value).String())
+ }
+ })
+ }
+
+ t.Run("Binary", func(t *testing.T) {
+ s, err := MakeLiteral([]byte{'h'})
+ assert.NoError(t, err)
+ assert.Equal(t, []byte{'h'}, s.GetScalar().GetBinary().GetValue())
+ })
+
+ t.Run("Blob", func(t *testing.T) {
+ l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Blob{}})
+ assert.NoError(t, err)
+ assert.Equal(t, "*core.Scalar_Blob", reflect.TypeOf(l.GetScalar().Value).String())
+ })
+
+ t.Run("Collection", func(t *testing.T) {
+ l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_CollectionType{CollectionType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}})
+ assert.NoError(t, err)
+ assert.Equal(t, "*core.LiteralCollection", reflect.TypeOf(l.GetCollection()).String())
+ })
+
+ t.Run("Map", func(t *testing.T) {
+ l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_MapValueType{MapValueType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}})
+ assert.NoError(t, err)
+ assert.Equal(t, "*core.LiteralMap", reflect.TypeOf(l.GetMap()).String())
+ })
+
+ t.Run("error", func(t *testing.T) {
+ l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Simple{
+ Simple: core.SimpleType_ERROR,
+ }})
+ assert.NoError(t, err)
+ assert.NotNil(t, l.GetScalar().GetError())
+ })
+
+ t.Run("binary", func(t *testing.T) {
+ l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Simple{
+ Simple: core.SimpleType_BINARY,
+ }})
+ assert.NoError(t, err)
+ assert.NotNil(t, l.GetScalar().GetBinary())
+ assert.NotNil(t, l.GetScalar().GetBinary().GetValue())
+ assert.NotNil(t, l.GetScalar().GetBinary().GetTag())
+ })
+
+ t.Run("struct", func(t *testing.T) {
+ l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Simple{
+ Simple: core.SimpleType_STRUCT,
+ }})
+ assert.NoError(t, err)
+ assert.NotNil(t, l.GetScalar().GetGeneric())
+ })
+
+ t.Run("enum", func(t *testing.T) {
+ l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_EnumType{
+ EnumType: &core.EnumType{Values: []string{"x", "y", "z"}},
+ }})
+ assert.NoError(t, err)
+ assert.NotNil(t, l.GetScalar().GetPrimitive().GetStringValue())
+ expected := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "x"}}}}}}
+ assert.Equal(t, expected, l)
+ })
+
+ t.Run("union", func(t *testing.T) {
+ l, err := MakeDefaultLiteralForType(
+ &core.LiteralType{
+ Type: &core.LiteralType_UnionType{
+ UnionType: &core.UnionType{
+ Variants: []*core.LiteralType{
+ {Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}},
+ {Type: &core.LiteralType_Simple{Simple: core.SimpleType_FLOAT}},
+ },
+ },
+ },
+ },
+ )
+ assert.NoError(t, err)
+ assert.Equal(t, "*core.Union", reflect.TypeOf(l.GetScalar().GetUnion()).String())
+ })
+}
+
+func TestMustMakeDefaultLiteralForType(t *testing.T) {
+ t.Run("error", func(t *testing.T) {
+ assert.Panics(t, func() {
+ MustMakeDefaultLiteralForType(nil)
+ })
+ })
+
+ t.Run("Blob", func(t *testing.T) {
+ l := MustMakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Blob{}})
+ assert.Equal(t, "*core.Scalar_Blob", reflect.TypeOf(l.GetScalar().Value).String())
+ })
+}
+
+func TestMakePrimitiveForType(t *testing.T) {
+ n := time.Now()
+ type args struct {
+ t core.SimpleType
+ s string
+ }
+ tests := []struct {
+ name string
+ args args
+ want *core.Primitive
+ wantErr bool
+ }{
+ {"error-type", args{core.SimpleType_NONE, "x"}, nil, true},
+
+ {"error-int", args{core.SimpleType_INTEGER, "x"}, nil, true},
+ {"int", args{core.SimpleType_INTEGER, "1"}, MustMakePrimitive(1), false},
+
+ {"error-bool", args{core.SimpleType_BOOLEAN, "x"}, nil, true},
+ {"bool", args{core.SimpleType_BOOLEAN, "true"}, MustMakePrimitive(true), false},
+
+ {"error-float", args{core.SimpleType_FLOAT, "x"}, nil, true},
+ {"float", args{core.SimpleType_FLOAT, "3.1416"}, MustMakePrimitive(3.1416), false},
+
+ {"string", args{core.SimpleType_STRING, "string"}, MustMakePrimitive("string"), false},
+
+ {"error-dt", args{core.SimpleType_DATETIME, "x"}, nil, true},
+ {"dt", args{core.SimpleType_DATETIME, n.Format(time.RFC3339Nano)}, MustMakePrimitive(n), false},
+
+ {"error-dur", args{core.SimpleType_DURATION, "x"}, nil, true},
+ {"dur", args{core.SimpleType_DURATION, time.Hour.String()}, MustMakePrimitive(time.Hour), false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := MakePrimitiveForType(tt.args.t, tt.args.s)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("MakePrimitiveForType() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("MakePrimitiveForType() got = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestMakeLiteralForSimpleType(t *testing.T) {
+ type args struct {
+ t core.SimpleType
+ s string
+ }
+ tests := []struct {
+ name string
+ args args
+ want *core.Literal
+ wantErr bool
+ }{
+ {"error-int", args{core.SimpleType_INTEGER, "x"}, nil, true},
+ {"int", args{core.SimpleType_INTEGER, "1"}, MustMakeLiteral(1), false},
+
+ {"error-struct", args{core.SimpleType_STRUCT, "x"}, nil, true},
+ {"struct", args{core.SimpleType_STRUCT, `{"x": 1}`}, MustMakeLiteral(&structpb.Struct{Fields: map[string]*structpb.Value{"x": {Kind: &structpb.Value_NumberValue{NumberValue: 1}}}}), false},
+
+ {"bin", args{core.SimpleType_BINARY, "x"}, MustMakeLiteral([]byte("x")), false},
+
+ {"error", args{core.SimpleType_ERROR, "err"}, MustMakeLiteral(&core.Error{Message: "err"}), false},
+
+ {"none", args{core.SimpleType_NONE, "null"}, MustMakeLiteral(nil), false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := MakeLiteralForSimpleType(tt.args.t, tt.args.s)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("MakeLiteralForSimpleType() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ if diff := deep.Equal(tt.want, got); diff != nil {
+ t.Errorf("MakeLiteralForSimpleType() got = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestMakeLiteralForBlob(t *testing.T) {
+ type args struct {
+ path storage.DataReference
+ isDir bool
+ format string
+ }
+ tests := []struct {
+ name string
+ args args
+ want *core.Blob
+ }{
+ {"simple-key", args{path: "/key", isDir: false, format: "xyz"}, &core.Blob{Uri: "/key", Metadata: &core.BlobMetadata{Type: &core.BlobType{Format: "xyz", Dimensionality: core.BlobType_SINGLE}}}},
+ {"simple-dir", args{path: "/key", isDir: true, format: "xyz"}, &core.Blob{Uri: "/key", Metadata: &core.BlobMetadata{Type: &core.BlobType{Format: "xyz", Dimensionality: core.BlobType_MULTIPART}}}},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := MakeLiteralForBlob(tt.args.path, tt.args.isDir, tt.args.format); !reflect.DeepEqual(got.GetScalar().GetBlob(), tt.want) {
+ t.Errorf("MakeLiteralForBlob() = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
+
+func TestMakeLiteralForType(t *testing.T) {
+ t.Run("SimpleInteger", func(t *testing.T) {
+ var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}
+ val, err := MakeLiteralForType(literalType, 1)
+ assert.NoError(t, err)
+ literalVal := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_Integer{Integer: 1}}}}}}
+ expectedVal, _ := ExtractFromLiteral(literalVal)
+ actualVal, _ := ExtractFromLiteral(val)
+ assert.Equal(t, expectedVal, actualVal)
+ })
+
+ t.Run("IntegerComingInAsFloatOverFlow", func(t *testing.T) {
+ var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}
+ _, err := MakeLiteralForType(literalType, 8.888888e+19)
+ assert.NotNil(t, err)
+ numError := &strconv.NumError{
+ Func: "ParseInt",
+ Num: "88888880000000000000",
+ Err: fmt.Errorf("value out of range"),
+ }
+ parseIntError := errors.WithMessage(numError, "failed to parse integer value")
+ assert.Equal(t, errors.WithStack(parseIntError).Error(), err.Error())
+ })
+
+ t.Run("IntegerComingInAsFloat", func(t *testing.T) {
+ var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}
+ val, err := MakeLiteralForType(literalType, 8.888888e+18)
+ assert.NoError(t, err)
+ literalVal := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_Integer{Integer: 8.888888e+18}}}}}}
+ expectedVal, _ := ExtractFromLiteral(literalVal)
+ actualVal, _ := ExtractFromLiteral(val)
+ assert.Equal(t, expectedVal, actualVal)
+ })
+
+ t.Run("SimpleFloat", func(t *testing.T) {
+ var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_FLOAT}}
+ val, err := MakeLiteralForType(literalType, 1)
+ assert.NoError(t, err)
+ literalVal := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_FloatValue{FloatValue: 1.0}}}}}}
+ expectedVal, _ := ExtractFromLiteral(literalVal)
+ actualVal, _ := ExtractFromLiteral(val)
+ assert.Equal(t, expectedVal, actualVal)
+ })
+
+ t.Run("Generic", func(t *testing.T) {
+ os.Setenv(FlyteUseOldDcFormat, "true")
+ literalVal := map[string]interface{}{
+ "x": 1,
+ "y": "ystringvalue",
+ }
+ var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRUCT}}
+ lit, err := MakeLiteralForType(literalType, literalVal)
+ assert.NoError(t, err)
+ extractedLiteralVal, err := ExtractFromLiteral(lit)
+ assert.NoError(t, err)
+ fieldsMap := map[string]*structpb.Value{
+ "x": {
+ Kind: &structpb.Value_NumberValue{NumberValue: 1},
+ },
+ "y": {
+ Kind: &structpb.Value_StringValue{StringValue: "ystringvalue"},
+ },
+ }
+ expectedStructVal := &structpb.Struct{
+ Fields: fieldsMap,
+ }
+ extractedStructValue := extractedLiteralVal.(*structpb.Struct)
+ assert.Equal(t, len(expectedStructVal.Fields), len(extractedStructValue.Fields))
+ for key, val := range expectedStructVal.Fields {
+ assert.Equal(t, val.Kind, extractedStructValue.Fields[key].Kind)
+ }
+ os.Unsetenv(FlyteUseOldDcFormat)
+ })
+
+ t.Run("SimpleBinary", func(t *testing.T) {
+ // We compare the deserialized values instead of the raw msgpack bytes because Go does not guarantee the order
+ // of map keys during serialization. This means that while the serialized bytes may differ, the deserialized
+ // values should be logically equivalent.
+
+ var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRUCT}}
+ v := map[string]interface{}{
+ "a": int64(1),
+ "b": 3.14,
+ "c": "example_string",
+ "d": map[string]interface{}{
+ "1": int64(100),
+ "2": int64(200),
+ },
+ "e": map[string]interface{}{
+ "a": int64(1),
+ "b": 3.14,
+ },
+ "f": []string{"a", "b", "c"},
+ }
+
+ val, err := MakeLiteralForType(literalType, v)
+ assert.NoError(t, err)
+
+ msgpackBytes, err := msgpack.Marshal(v)
+ assert.NoError(t, err)
+
+ literalVal := &core.Literal{
+ Value: &core.Literal_Scalar{
+ Scalar: &core.Scalar{
+ Value: &core.Scalar_Binary{
+ Binary: &core.Binary{
+ Value: msgpackBytes,
+ Tag: MESSAGEPACK,
+ },
+ },
+ },
+ },
+ }
+
+ expectedLiteralVal, err := ExtractFromLiteral(literalVal)
+ assert.NoError(t, err)
+ actualLiteralVal, err := ExtractFromLiteral(val)
+ assert.NoError(t, err)
+
+ // Check if the extracted value is of type *core.Binary (not []byte)
+ expectedBinary, ok := expectedLiteralVal.(*core.Binary)
+ assert.True(t, ok, "expectedLiteralVal is not of type *core.Binary")
+ actualBinary, ok := actualLiteralVal.(*core.Binary)
+ assert.True(t, ok, "actualLiteralVal is not of type *core.Binary")
+
+ // Now check if the Binary values match
+ var expectedVal, actualVal map[string]interface{}
+ err = msgpack.Unmarshal(expectedBinary.Value, &expectedVal)
+ assert.NoError(t, err)
+ err = msgpack.Unmarshal(actualBinary.Value, &actualVal)
+ assert.NoError(t, err)
+
+ // Finally, assert that the deserialized values are equal
+ assert.Equal(t, expectedVal, actualVal)
+ })
+
+ t.Run("ArrayStrings", func(t *testing.T) {
+ var literalType = &core.LiteralType{Type: &core.LiteralType_CollectionType{
+ CollectionType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}}}
+ strArray := []interface{}{"hello", "world"}
+ val, err := MakeLiteralForType(literalType, strArray)
+ assert.NoError(t, err)
+ literalVal1 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "hello"}}}}}}
+ literalVal2 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world"}}}}}}
+ literalCollection := []*core.Literal{literalVal1, literalVal2}
+ literalVal := &core.Literal{Value: &core.Literal_Collection{Collection: &core.LiteralCollection{Literals: literalCollection}}}
+ expectedVal, _ := ExtractFromLiteral(literalVal)
+ actualVal, _ := ExtractFromLiteral(val)
+ assert.Equal(t, expectedVal, actualVal)
+ })
+
+ t.Run("ArrayOfArrayStringsNotSupported", func(t *testing.T) {
+ var literalType = &core.LiteralType{Type: &core.LiteralType_CollectionType{
+ CollectionType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}}}
+ strArrayOfArray := [][]interface{}{{"hello1", "world1"}, {"hello2", "world2"}}
+ _, err := MakeLiteralForType(literalType, strArrayOfArray)
+ expectedErrorf := fmt.Errorf("collection type expected but found [][]interface {}")
+ assert.Equal(t, expectedErrorf, err)
+ })
+
+ t.Run("ArrayOfArrayStringsTypeErasure", func(t *testing.T) {
+ var collectionType = &core.LiteralType{Type: &core.LiteralType_CollectionType{
+ CollectionType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}}}
+ var literalType = &core.LiteralType{Type: &core.LiteralType_CollectionType{
+ CollectionType: collectionType}}
+
+ createList1 := func() interface{} {
+ return []interface{}{"hello1", "world1"}
+ }
+ createList2 := func() interface{} {
+ return []interface{}{"hello2", "world2"}
+ }
+ createNestedList := func() interface{} {
+ return []interface{}{createList1(), createList2()}
+ }
+ var strArrayOfArray = createNestedList()
+ val, err := MakeLiteralForType(literalType, strArrayOfArray)
+ assert.NoError(t, err)
+ literalVal11 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "hello1"}}}}}}
+ literalVal12 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world1"}}}}}}
+ literalCollection1Val := []*core.Literal{literalVal11, literalVal12}
+
+ literalCollection1 := &core.Literal{Value: &core.Literal_Collection{Collection: &core.LiteralCollection{Literals: literalCollection1Val}}}
+
+ literalVal21 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "hello2"}}}}}}
+ literalVal22 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world2"}}}}}}
+ literalCollection2Val := []*core.Literal{literalVal21, literalVal22}
+ literalCollection2 := &core.Literal{Value: &core.Literal_Collection{Collection: &core.LiteralCollection{Literals: literalCollection2Val}}}
+ literalCollection := []*core.Literal{literalCollection1, literalCollection2}
+
+ literalVal := &core.Literal{Value: &core.Literal_Collection{Collection: &core.LiteralCollection{Literals: literalCollection}}}
+ expectedVal, _ := ExtractFromLiteral(literalVal)
+ actualVal, _ := ExtractFromLiteral(val)
+ assert.Equal(t, expectedVal, actualVal)
+ })
+
+ t.Run("MapStrings", func(t *testing.T) {
+ var literalType = &core.LiteralType{Type: &core.LiteralType_MapValueType{
+ MapValueType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}}}
+ mapVal := map[string]interface{}{"hello1": "world1", "hello2": "world2"}
+ val, err := MakeLiteralForType(literalType, mapVal)
+ assert.NoError(t, err)
+ literalVal1 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world1"}}}}}}
+ literalVal2 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world2"}}}}}}
+ literalMapVal := map[string]*core.Literal{"hello1": literalVal1, "hello2": literalVal2}
+ literalVal := &core.Literal{Value: &core.Literal_Map{Map: &core.LiteralMap{Literals: literalMapVal}}}
+ expectedVal, _ := ExtractFromLiteral(literalVal)
+ actualVal, _ := ExtractFromLiteral(val)
+ assert.Equal(t, expectedVal, actualVal)
+ })
+
+ t.Run("MapArrayOfStringsFail", func(t *testing.T) {
+ var literalType = &core.LiteralType{Type: &core.LiteralType_MapValueType{
+ MapValueType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}}}
+ strArray := map[string][]interface{}{"hello1": {"world11", "world12"}, "hello2": {"world21", "world22"}}
+ _, err := MakeLiteralForType(literalType, strArray)
+ expectedErrorf := fmt.Errorf("map value types can only be of type map[string]interface{}, but found map[string][]interface {}")
+ assert.Equal(t, expectedErrorf, err)
+ })
+
+ t.Run("MapArrayOfStringsTypeErasure", func(t *testing.T) {
+ var collectionType = &core.LiteralType{Type: &core.LiteralType_CollectionType{
+ CollectionType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}}}
+ var literalType = &core.LiteralType{Type: &core.LiteralType_MapValueType{
+ MapValueType: collectionType}}
+ createList1 := func() interface{} {
+ return []interface{}{"world11", "world12"}
+ }
+ createList2 := func() interface{} {
+ return []interface{}{"world21", "world22"}
+ }
+ strArray := map[string]interface{}{"hello1": createList1(), "hello2": createList2()}
+ val, err := MakeLiteralForType(literalType, strArray)
+ assert.NoError(t, err)
+ literalVal11 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world11"}}}}}}
+ literalVal12 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world12"}}}}}}
+ literalCollection1 := []*core.Literal{literalVal11, literalVal12}
+ literalVal1 := &core.Literal{Value: &core.Literal_Collection{Collection: &core.LiteralCollection{Literals: literalCollection1}}}
+ literalVal21 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world21"}}}}}}
+ literalVal22 := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "world22"}}}}}}
+ literalCollection2 := []*core.Literal{literalVal21, literalVal22}
+ literalVal2 := &core.Literal{Value: &core.Literal_Collection{Collection: &core.LiteralCollection{Literals: literalCollection2}}}
+ literalMapVal := map[string]*core.Literal{"hello1": literalVal1, "hello2": literalVal2}
+ literalVal := &core.Literal{Value: &core.Literal_Map{Map: &core.LiteralMap{Literals: literalMapVal}}}
+ expectedVal, _ := ExtractFromLiteral(literalVal)
+ actualVal, _ := ExtractFromLiteral(val)
+ assert.Equal(t, expectedVal, actualVal)
+ })
+
+ t.Run("Schema", func(t *testing.T) {
+ var schemaColumns []*core.SchemaType_SchemaColumn
+ schemaColumns = append(schemaColumns, &core.SchemaType_SchemaColumn{
+ Name: "Price",
+ Type: core.SchemaType_SchemaColumn_FLOAT,
+ })
+ var literalType = &core.LiteralType{Type: &core.LiteralType_Schema{Schema: &core.SchemaType{
+ Columns: schemaColumns,
+ }}}
+
+ expectedLV := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Schema{
+ Schema: &core.Schema{
+ Uri: "s3://blah/blah/blah",
+ Type: &core.SchemaType{
+ Columns: schemaColumns,
+ },
+ },
+ },
+ }}}
+ lv, err := MakeLiteralForType(literalType, "s3://blah/blah/blah")
+ assert.NoError(t, err)
+
+ assert.Equal(t, expectedLV, lv)
+
+ expectedVal, err := ExtractFromLiteral(expectedLV)
+ assert.NoError(t, err)
+ actualVal, err := ExtractFromLiteral(lv)
+ assert.NoError(t, err)
+ assert.Equal(t, expectedVal, actualVal)
+ })
+
+ t.Run("Blob", func(t *testing.T) {
+ var literalType = &core.LiteralType{Type: &core.LiteralType_Blob{Blob: &core.BlobType{
+ Dimensionality: core.BlobType_SINGLE,
+ }}}
+ expectedLV := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Blob{
+ Blob: &core.Blob{
+ Uri: "s3://blah/blah/blah",
+ Metadata: &core.BlobMetadata{
+ Type: &core.BlobType{
+ Dimensionality: core.BlobType_SINGLE,
+ },
+ },
+ },
+ },
+ }}}
+ lv, err := MakeLiteralForType(literalType, "s3://blah/blah/blah")
+ assert.NoError(t, err)
+
+ assert.Equal(t, expectedLV, lv)
+
+ expectedVal, err := ExtractFromLiteral(expectedLV)
+ assert.NoError(t, err)
+ actualVal, err := ExtractFromLiteral(lv)
+ assert.NoError(t, err)
+ assert.Equal(t, expectedVal, actualVal)
+ })
+
+ t.Run("MultipartBlob", func(t *testing.T) {
+ var literalType = &core.LiteralType{Type: &core.LiteralType_Blob{Blob: &core.BlobType{
+ Dimensionality: core.BlobType_MULTIPART,
+ }}}
+ expectedLV := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Blob{
+ Blob: &core.Blob{
+ Uri: "s3://blah/blah/blah",
+ Metadata: &core.BlobMetadata{
+ Type: &core.BlobType{
+ Dimensionality: core.BlobType_MULTIPART,
+ },
+ },
+ },
+ },
+ }}}
+ lv, err := MakeLiteralForType(literalType, "s3://blah/blah/blah")
+ assert.NoError(t, err)
+
+ assert.Equal(t, expectedLV, lv)
+
+ expectedVal, err := ExtractFromLiteral(expectedLV)
+ assert.NoError(t, err)
+ actualVal, err := ExtractFromLiteral(lv)
+ assert.NoError(t, err)
+ assert.Equal(t, expectedVal, actualVal)
+ })
+
+ t.Run("enumtype-nil", func(t *testing.T) {
+ var literalType = &core.LiteralType{Type: &core.LiteralType_EnumType{EnumType: &core.EnumType{}}}
+ _, err := MakeLiteralForType(literalType, nil)
+ assert.Error(t, err)
+ _, err = MakeLiteralForType(literalType, "")
+ assert.Error(t, err)
+ })
+
+ t.Run("enumtype-happy", func(t *testing.T) {
+ var literalType = &core.LiteralType{Type: &core.LiteralType_EnumType{EnumType: &core.EnumType{Values: []string{"x", "y", "z"}}}}
+ expected := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_StringValue{StringValue: "x"}}}}}}
+ v, err := MakeLiteralForType(literalType, "x")
+ assert.NoError(t, err)
+ assert.Equal(t, expected, v)
+ _, err = MakeLiteralForType(literalType, "")
+ assert.Error(t, err)
+ })
+
+ t.Run("enumtype-illegal-val", func(t *testing.T) {
+ var literalType = &core.LiteralType{Type: &core.LiteralType_EnumType{EnumType: &core.EnumType{Values: []string{"x", "y", "z"}}}}
+ _, err := MakeLiteralForType(literalType, "m")
+ assert.Error(t, err)
+ })
+
+ t.Run("Nil string", func(t *testing.T) {
+ var literalType = &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}
+ l, err := MakeLiteralForType(literalType, nil)
+ assert.NoError(t, err)
+ assert.Equal(t, "", l.GetScalar().GetPrimitive().GetStringValue())
+ l, err = MakeLiteralForType(literalType, "")
+ assert.NoError(t, err)
+ assert.Equal(t, "", l.GetScalar().GetPrimitive().GetStringValue())
+ })
+
+ t.Run("Structured Data Set", func(t *testing.T) {
+ var dataSetColumns []*core.StructuredDatasetType_DatasetColumn
+ dataSetColumns = append(dataSetColumns, &core.StructuredDatasetType_DatasetColumn{
+ Name: "Price",
+ LiteralType: &core.LiteralType{
+ Type: &core.LiteralType_Simple{
+ Simple: core.SimpleType_FLOAT,
+ },
+ },
+ })
+ var literalType = &core.LiteralType{Type: &core.LiteralType_StructuredDatasetType{StructuredDatasetType: &core.StructuredDatasetType{
+ Columns: dataSetColumns,
+ Format: "testFormat",
+ }}}
+
+ expectedLV := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_StructuredDataset{
+ StructuredDataset: &core.StructuredDataset{
+ Uri: "s3://blah/blah/blah",
+ Metadata: &core.StructuredDatasetMetadata{
+ StructuredDatasetType: &core.StructuredDatasetType{
+ Columns: dataSetColumns,
+ Format: "testFormat",
+ },
+ },
+ },
+ },
+ }}}
+ lv, err := MakeLiteralForType(literalType, "s3://blah/blah/blah")
+ assert.NoError(t, err)
+
+ assert.Equal(t, expectedLV, lv)
+
+ expectedVal, err := ExtractFromLiteral(expectedLV)
+ assert.NoError(t, err)
+ actualVal, err := ExtractFromLiteral(lv)
+ assert.NoError(t, err)
+ assert.Equal(t, expectedVal, actualVal)
+ })
+
+ t.Run("Union", func(t *testing.T) {
+ var literalType = &core.LiteralType{
+ Type: &core.LiteralType_UnionType{
+ UnionType: &core.UnionType{
+ Variants: []*core.LiteralType{
+ {Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}},
+ {Type: &core.LiteralType_Simple{Simple: core.SimpleType_FLOAT}},
+ },
+ },
+ },
+ }
+ expectedLV := &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Union{
+ Union: &core.Union{
+ Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_FLOAT}},
+ Value: &core.Literal{Value: &core.Literal_Scalar{Scalar: &core.Scalar{
+ Value: &core.Scalar_Primitive{Primitive: &core.Primitive{Value: &core.Primitive_FloatValue{FloatValue: 0.1}}}}}},
+ },
+ },
+ }}}
+ lv, err := MakeLiteralForType(literalType, float64(0.1))
+ assert.NoError(t, err)
+ assert.Equal(t, expectedLV, lv)
+ expectedVal, err := ExtractFromLiteral(expectedLV)
+ assert.NoError(t, err)
+ actualVal, err := ExtractFromLiteral(lv)
+ assert.NoError(t, err)
+ assert.Equal(t, expectedVal, actualVal)
+ })
+}
diff --git a/flyteidl2/cluster/payload.proto b/flyteidl2/cluster/payload.proto
new file mode 100644
index 00000000000..fa996cab242
--- /dev/null
+++ b/flyteidl2/cluster/payload.proto
@@ -0,0 +1,41 @@
+syntax = "proto3";
+
+package flyteidl2.cluster;
+
+import "buf/validate/validate.proto";
+import "flyteidl2/app/app_definition.proto";
+import "flyteidl2/common/identifier.proto";
+import "flyteidl2/task/task_definition.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/cluster";
+
+message SelectClusterRequest {
+ oneof resource {
+ option (buf.validate.oneof).required = true;
+ flyteidl2.common.OrgIdentifier org_id = 1;
+ flyteidl2.common.ProjectIdentifier project_id = 2;
+ flyteidl2.task.TaskIdentifier task_id = 3;
+ flyteidl2.common.ActionIdentifier action_id = 4;
+ flyteidl2.common.ActionAttemptIdentifier action_attempt_id = 5;
+ flyteidl2.app.Identifier app_id = 6;
+ }
+
+ enum Operation {
+ OPERATION_UNSPECIFIED = 0;
+ OPERATION_CREATE_UPLOAD_LOCATION = 1;
+ OPERATION_UPLOAD_INPUTS = 2;
+ OPERATION_GET_ACTION_DATA = 3;
+ OPERATION_QUERY_RANGE_METRICS = 4;
+ OPERATION_CREATE_DOWNLOAD_LINK = 5;
+ OPERATION_TAIL_LOGS = 6;
+ OPERATION_GET_ACTION_ATTEMPT_METRICS = 7;
+ }
+
+ Operation operation = 8 [(buf.validate.field).enum = {
+ not_in: [0]
+ }];
+}
+
+message SelectClusterResponse {
+ string cluster_endpoint = 1;
+}
diff --git a/flyteidl2/cluster/service.proto b/flyteidl2/cluster/service.proto
new file mode 100644
index 00000000000..f81236a5f61
--- /dev/null
+++ b/flyteidl2/cluster/service.proto
@@ -0,0 +1,11 @@
+syntax = "proto3";
+
+package flyteidl2.cluster;
+
+import "flyteidl2/cluster/payload.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/cluster";
+
+service ClusterService {
+ rpc SelectCluster(SelectClusterRequest) returns (SelectClusterResponse) {}
+}
diff --git a/flyteidl2/common/authorization.proto b/flyteidl2/common/authorization.proto
new file mode 100644
index 00000000000..bac5b9259ff
--- /dev/null
+++ b/flyteidl2/common/authorization.proto
@@ -0,0 +1,97 @@
+syntax = "proto3";
+
+package flyteidl2.common;
+
+import "buf/validate/validate.proto";
+import "flyteidl2/common/identifier.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common";
+
+message Organization {
+ string name = 1 [(buf.validate.field).string.min_len = 1];
+}
+
+message Domain {
+ string name = 1;
+ Organization organization = 2 [(buf.validate.field).required = true];
+}
+
+message Project {
+ string name = 1 [(buf.validate.field).string.min_len = 1];
+ Domain domain = 2 [(buf.validate.field).required = true];
+}
+
+message Workflow {
+ string name = 1 [(buf.validate.field).string.min_len = 1];
+ Project project = 2 [(buf.validate.field).required = true];
+}
+
+message LaunchPlan {
+ string name = 1 [(buf.validate.field).string.min_len = 1];
+ Project project = 2 [(buf.validate.field).required = true];
+}
+
+message Resource {
+ oneof resource {
+ Organization organization = 1;
+ Domain domain = 2;
+ Project project = 3;
+ Workflow workflow = 4;
+ LaunchPlan launch_plan = 5;
+ ClusterIdentifier cluster = 6;
+ }
+}
+
+enum Action {
+ ACTION_NONE = 0;
+ ACTION_CREATE = 1 [deprecated = true];
+ ACTION_READ = 2 [deprecated = true];
+ ACTION_UPDATE = 3 [deprecated = true];
+ ACTION_DELETE = 4 [deprecated = true];
+
+ // Read Flyte workflows, tasks and launch plans
+ ACTION_VIEW_FLYTE_INVENTORY = 5;
+
+ // View Flyte executions
+ ACTION_VIEW_FLYTE_EXECUTIONS = 6;
+
+ // Register new versions of Flyte workflows, tasks and launch plans
+ ACTION_REGISTER_FLYTE_INVENTORY = 7;
+
+ // Create new Flyte workflow and task executions
+ ACTION_CREATE_FLYTE_EXECUTIONS = 8;
+
+ // Create new projects and update project descriptions
+ ACTION_ADMINISTER_PROJECT = 9;
+
+ // Add users, roles and update role assignments.
+ ACTION_MANAGE_PERMISSIONS = 10;
+
+ // Manage billing, account-wide settings
+ ACTION_ADMINISTER_ACCOUNT = 11;
+
+ // Operations for clusters
+ ACTION_MANAGE_CLUSTER = 12;
+
+ // Edit execution related attributes, including TASK_RESOURCE, WORKFLOW_EXECUTION_CONFIG, and EXTERNAL_RESOURCE
+ ACTION_EDIT_EXECUTION_RELATED_ATTRIBUTES = 13;
+
+ // Edit cluster related attributes, including CLUSTER_RESOURCE and CLUSTER_ASSIGNMENT
+ ACTION_EDIT_CLUSTER_RELATED_ATTRIBUTES = 14;
+
+ // Edit unused attributes, including EXECUTION_QUEUE, EXECUTION_CLUSTER_LABEL, QUALITY_OF_SERVICE_SPECIFICATION, and PLUGIN_OVERRIDE
+ ACTION_EDIT_UNUSED_ATTRIBUTES = 15;
+
+ // View system logs
+ ACTION_SUPPORT_SYSTEM_LOGS = 16;
+
+ // View identities, this includes human users and machine apps (client creds)
+ ACTION_VIEW_IDENTITIES = 17;
+}
+
+// Defines a set of allowed actions on a specific authorization resource.
+message Permission {
+ Resource resource = 1;
+
+ repeated Action actions = 2;
+}
diff --git a/flyteidl2/common/configuration.proto b/flyteidl2/common/configuration.proto
new file mode 100644
index 00000000000..1d47395336d
--- /dev/null
+++ b/flyteidl2/common/configuration.proto
@@ -0,0 +1,26 @@
+syntax = "proto3";
+
+package flyteidl2.common;
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common";
+
+// The source of an attribute. We may have other sources in the future.
+enum AttributesSource {
+ // The source is unspecified.
+ SOURCE_UNSPECIFIED = 0;
+
+ // The configuration is a global configuration.
+ GLOBAL = 1;
+
+ // The configuration is a domain configuration.
+ DOMAIN = 2;
+
+ // The configuration is a project configuration.
+ PROJECT = 3;
+
+ // The configuration is a project-domain configuration.
+ PROJECT_DOMAIN = 4;
+
+ // The configuration is a org configuration.
+ ORG = 5;
+}
diff --git a/flyteidl2/common/identifier.proto b/flyteidl2/common/identifier.proto
new file mode 100644
index 00000000000..685c396cb25
--- /dev/null
+++ b/flyteidl2/common/identifier.proto
@@ -0,0 +1,156 @@
+syntax = "proto3";
+
+package flyteidl2.common;
+
+import "buf/validate/validate.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common";
+
+message ProjectIdentifier {
+ string organization = 1 [(buf.validate.field).string.min_len = 1];
+ string domain = 2 [(buf.validate.field).string.min_len = 1];
+ string name = 3 [(buf.validate.field).string.min_len = 1];
+}
+
+message ClusterIdentifier {
+ string organization = 1;
+ string name = 2 [(buf.validate.field).string.min_len = 1];
+}
+
+message ClusterPoolIdentifier {
+ string organization = 1;
+ string name = 2;
+}
+
+message ClusterConfigIdentifier {
+ string organization = 1 [(buf.validate.field).string.min_len = 1];
+ string id = 2 [(buf.validate.field).string.min_len = 1];
+}
+
+message ClusterNodepoolIdentifier {
+ string organization = 1;
+ string cluster_name = 2 [(buf.validate.field).string.min_len = 1];
+ string name = 3 [(buf.validate.field).string.min_len = 1];
+}
+
+message UserIdentifier {
+ string subject = 1 [(buf.validate.field).string.min_len = 1];
+}
+
+message ApplicationIdentifier {
+ string subject = 1 [(buf.validate.field).string.min_len = 1];
+}
+
+message RoleIdentifier {
+ string organization = 1;
+
+ // Unique name for this role within the organization
+ string name = 2 [(buf.validate.field).string.min_len = 1];
+}
+
+message OrgIdentifier {
+ string name = 1 [(buf.validate.field).string = {
+ min_len: 1
+ max_len: 63
+ pattern: "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"
+ }];
+}
+
+message ManagedClusterIdentifier {
+ reserved 1;
+ string name = 2 [(buf.validate.field).string.min_len = 1];
+ OrgIdentifier org = 3 [(buf.validate.field).required = true];
+}
+
+message PolicyIdentifier {
+ string organization = 1;
+
+ // Unique name for this policy within the organization
+ string name = 2 [(buf.validate.field).string.min_len = 1];
+}
+
+// Unique identifier of a run.
+message RunIdentifier {
+ // Org this run belongs to.
+ string org = 1 [
+ (buf.validate.field).string.min_len = 1,
+ (buf.validate.field).string.max_len = 63
+ ];
+
+ // Project this run belongs to.
+ string project = 2 [
+ (buf.validate.field).string.min_len = 1,
+ (buf.validate.field).string.max_len = 63
+ ];
+
+ // Domain this run belongs to.
+ string domain = 3 [
+ (buf.validate.field).string.min_len = 1,
+ (buf.validate.field).string.max_len = 63
+ ];
+
+ // Name of the run. Must be unique across all runs in this org, project, and domain pairing.
+ string name = 4 [
+ (buf.validate.field).string.min_len = 1,
+ (buf.validate.field).string.max_len = 30
+ ];
+}
+
+// Unique identifier of an action.
+message ActionIdentifier {
+ // Identifier for the run.
+ RunIdentifier run = 1 [(buf.validate.field).required = true];
+
+ // Name of the action. Must be unique within the run.
+ string name = 2 [
+ (buf.validate.field).string.min_len = 1,
+ (buf.validate.field).string.max_len = 30
+ ];
+}
+
+// Unique identifier of a single action attempt
+message ActionAttemptIdentifier {
+ ActionIdentifier action_id = 1 [(buf.validate.field).required = true];
+
+ uint32 attempt = 2 [(buf.validate.field).uint32.gt = 0];
+}
+
+// Identifies trigger within an org, project and domain
+message TriggerName {
+ // Org this trigger belongs to.
+ string org = 1 [
+ (buf.validate.field).string.min_len = 1,
+ (buf.validate.field).string.max_len = 63
+ ];
+
+ // Project this trigger belongs to.
+ string project = 2 [
+ (buf.validate.field).string.min_len = 1,
+ (buf.validate.field).string.max_len = 63
+ ];
+
+ // Domain this trigger belongs to.
+ string domain = 3 [
+ (buf.validate.field).string.min_len = 1,
+ (buf.validate.field).string.max_len = 63
+ ];
+
+ // Unique name of the trigger.
+ string name = 4 [
+ (buf.validate.field).string.min_len = 1,
+ (buf.validate.field).string.max_len = 255
+ ];
+
+ string task_name = 5 [
+ (buf.validate.field).string.min_len = 1,
+ (buf.validate.field).string.max_len = 255
+ ];
+}
+
+// Identifies a trigger revision within an org, project and domain
+message TriggerIdentifier {
+ TriggerName name = 1 [(buf.validate.field).required = true];
+
+ // Revision of the trigger.
+ uint64 revision = 2 [(buf.validate.field).uint64.gt = 0];
+}
diff --git a/flyteidl2/common/identity.proto b/flyteidl2/common/identity.proto
new file mode 100644
index 00000000000..6340f353e86
--- /dev/null
+++ b/flyteidl2/common/identity.proto
@@ -0,0 +1,64 @@
+syntax = "proto3";
+
+package flyteidl2.common;
+
+import "buf/validate/validate.proto";
+import "flyteidl2/common/identifier.proto";
+import "flyteidl2/common/policy.proto";
+import "flyteidl2/common/role.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common";
+
+// Encapsulates user profile details for a member of an organization.
+message User {
+ UserIdentifier id = 1;
+
+ UserSpec spec = 2;
+
+ repeated Role roles = 3 [deprecated = true];
+
+ repeated Policy policies = 4;
+}
+
+message UserSpec {
+ string first_name = 1;
+
+ string last_name = 2;
+
+ string email = 3;
+
+ string organization = 4;
+
+ string user_handle = 5;
+
+ repeated string groups = 6;
+
+ string photo_url = 7;
+}
+
+message Application {
+ ApplicationIdentifier id = 1;
+
+ AppSpec spec = 2;
+}
+
+message AppSpec {
+ string name = 1;
+
+ string organization = 2;
+}
+
+message EnrichedIdentity {
+ oneof principal {
+ option (buf.validate.oneof).required = true;
+ User user = 1;
+ Application application = 2;
+ }
+}
+
+message Identity {
+ oneof principal {
+ common.UserIdentifier user_id = 1;
+ common.ApplicationIdentifier application_id = 2;
+ }
+}
diff --git a/flyteidl2/common/list.proto b/flyteidl2/common/list.proto
new file mode 100644
index 00000000000..105f097b03b
--- /dev/null
+++ b/flyteidl2/common/list.proto
@@ -0,0 +1,81 @@
+syntax = "proto3";
+
+package flyteidl2.common;
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common";
+
+// Specifies sort ordering in a list request.
+message Sort {
+ enum Direction {
+ // By default, fields are sorted in descending order.
+ DESCENDING = 0;
+ ASCENDING = 1;
+ }
+
+ // Indicates an attribute to sort the response values.
+ // +required
+ string key = 1;
+
+ // Indicates the direction to apply sort key for response values.
+ // +optional
+ Direction direction = 2;
+}
+
+message ListRequest {
+ // Indicates the number of resources to be returned.
+ // +required
+ uint32 limit = 1;
+
+ // In the case of multiple pages of results, the server-provided token can be used to fetch the next page
+ // in a query.
+ // +optional
+ string token = 2;
+
+ // Deprecated, use sort_by_fields instead.
+ // Specifies how listed entities should be sorted in the response.
+ // +optional
+ Sort sort_by = 3 [deprecated = true];
+
+ // Indicates a list of filters. This field is used for grpc get requests.
+ // +optional
+ repeated Filter filters = 4;
+
+ // Indicates a raw list of filters passed as string.This field is used for REST get requests
+ // +optional
+ repeated string raw_filters = 5;
+
+ // Specifies how listed entities should be sorted in the response.
+ // Sort fields are applied in order.
+ // +optional
+ repeated Sort sort_by_fields = 6;
+}
+
+message Filter {
+ enum Function {
+ EQUAL = 0;
+ NOT_EQUAL = 1;
+ GREATER_THAN = 2;
+ GREATER_THAN_OR_EQUAL = 3;
+ LESS_THAN = 4;
+ LESS_THAN_OR_EQUAL = 5;
+ CONTAINS = 6; // Case sensitive contains function.
+ VALUE_IN = 7;
+
+ // NOT_CONTAINS = 8;
+ // VALUE_NOT_IN = 9;
+ // STARTS_WITH = 10;
+ // NOT_STARTS_WITH = 11;
+
+ ENDS_WITH = 12;
+ NOT_ENDS_WITH = 13;
+ CONTAINS_CASE_INSENSITIVE = 14; // Case insensitive contains function.
+ }
+
+ Function function = 1;
+
+ // e.g. name or version
+ string field = 2;
+
+ // Only in the case of a VALUE_IN function, values may contain multiple entries.
+ repeated string values = 3;
+}
diff --git a/flyteidl2/common/phase.proto b/flyteidl2/common/phase.proto
new file mode 100644
index 00000000000..98c5d0f5272
--- /dev/null
+++ b/flyteidl2/common/phase.proto
@@ -0,0 +1,42 @@
+syntax = "proto3";
+
+package flyteidl2.common;
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common";
+
+// ActionPhase represents the execution state of an action.
+//
+// Phase transitions follow this typical flow:
+// QUEUED -> WAITING_FOR_RESOURCES -> INITIALIZING -> RUNNING -> {SUCCEEDED|FAILED|ABORTED|TIMED_OUT}
+// Condition actions: PAUSED -> {SUCCEEDED(signal)|TIMED_OUT(timeout)|ABORTED(abort)}
+enum ActionPhase {
+ // Default/unknown phase
+ ACTION_PHASE_UNSPECIFIED = 0;
+
+ // Action has been accepted and is waiting to be scheduled
+ ACTION_PHASE_QUEUED = 1;
+
+ // Action is scheduled but waiting for compute resources to become available
+ ACTION_PHASE_WAITING_FOR_RESOURCES = 2;
+
+ // Resources have been allocated and the action is being set up
+ ACTION_PHASE_INITIALIZING = 3;
+
+ // Action is actively executing
+ ACTION_PHASE_RUNNING = 4;
+
+ // Action completed successfully
+ ACTION_PHASE_SUCCEEDED = 5;
+
+ // Action failed during execution
+ ACTION_PHASE_FAILED = 6;
+
+ // Action was manually terminated or cancelled
+ ACTION_PHASE_ABORTED = 7;
+
+ // Action exceeded its execution time limit
+ ACTION_PHASE_TIMED_OUT = 8;
+
+ // Action is paused and waiting for an external signal (condition actions)
+ ACTION_PHASE_PAUSED = 9;
+}
diff --git a/flyteidl2/common/policy.proto b/flyteidl2/common/policy.proto
new file mode 100644
index 00000000000..bf3fbc24028
--- /dev/null
+++ b/flyteidl2/common/policy.proto
@@ -0,0 +1,27 @@
+syntax = "proto3";
+
+package flyteidl2.common;
+
+import "buf/validate/validate.proto";
+import "flyteidl2/common/authorization.proto";
+import "flyteidl2/common/identifier.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common";
+
+// A policy is a collection of roles bound to a resource.
+message Policy {
+ PolicyIdentifier id = 1 [(buf.validate.field).required = true];
+
+ repeated PolicyBinding bindings = 2;
+
+ // Optional: human readable description
+ string description = 3;
+}
+
+// A policy binding represents a role (a set of actions) defined on a resource.
+message PolicyBinding {
+ // The role designates the permitted set of actions which can be applied to the resource.
+ RoleIdentifier role_id = 1 [(buf.validate.field).required = true];
+
+ common.Resource resource = 2 [(buf.validate.field).required = true];
+}
diff --git a/flyteidl2/common/role.proto b/flyteidl2/common/role.proto
new file mode 100644
index 00000000000..168655a8976
--- /dev/null
+++ b/flyteidl2/common/role.proto
@@ -0,0 +1,62 @@
+syntax = "proto3";
+
+package flyteidl2.common;
+
+import "buf/validate/validate.proto";
+import "flyteidl2/common/authorization.proto";
+import "flyteidl2/common/identifier.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common";
+
+// A role type is a short-hand for understanding the permissions associated with a role.
+// Boilerplate role types include a conventional collection of permissions
+// Custom role types include a user-defined collection of permissions
+enum RoleType {
+ // Default group. Not used in practice.
+ ROLE_TYPE_NONE = 0;
+ // The admin role has a collective set of permissions to do everything
+ ROLE_TYPE_ADMIN = 1;
+ // The contributor role has a collective set of permissions to view inventory, view executions, write inventory and create executions
+ ROLE_TYPE_CONTRIBUTOR = 2;
+ // The viewer role has a collective set of permissions to view inventory and view executions
+ ROLE_TYPE_VIEWER = 3;
+
+ // Represent a role with user-defined sets of permissions.
+ ROLE_TYPE_CUSTOM = 4;
+
+ // The role with permissions to administer a specific customer cluster.
+ ROLE_TYPE_CLUSTER_MANAGER = 5;
+
+ // Role with permissions specific to administer flyte project(s).
+ ROLE_TYPE_FLYTE_PROJECT_ADMIN = 6;
+
+ // The viewer role for serverless
+ ROLE_TYPE_SERVERLESS_VIEWER = 7;
+
+ // The contributor role for serverless
+ ROLE_TYPE_SERVERLESS_CONTRIBUTOR = 8;
+
+ // The support role would have contributor permissions plus the access to support endpoints
+ ROLE_TYPE_SUPPORT = 9;
+
+ // Internal system-provisioned role assigned to all identities by default.
+ // Grants baseline access required for platform functionality (e.g. image builder).
+ ROLE_TYPE_SYSTEM_PROVISIONED_ACCESS = 10;
+}
+
+message Role {
+ RoleIdentifier id = 1 [(buf.validate.field).required = true];
+
+ repeated Permission permissions = 2 [deprecated = true];
+
+ RoleSpec role_spec = 3;
+
+ RoleType role_type = 4;
+
+ repeated Action actions = 5;
+}
+
+message RoleSpec {
+ // Optional, human readable description for this role.
+ string description = 1;
+}
diff --git a/flyteidl2/common/run.proto b/flyteidl2/common/run.proto
new file mode 100644
index 00000000000..e162eb94a15
--- /dev/null
+++ b/flyteidl2/common/run.proto
@@ -0,0 +1,14 @@
+syntax = "proto3";
+
+package flyteidl2.common;
+
+import "buf/validate/validate.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common";
+
+// Captures required data to reference offloaded inputs.
+message OffloadedInputData {
+ string uri = 1 [(buf.validate.field).string.min_len = 1];
+
+ string inputs_hash = 2 [(buf.validate.field).string.min_len = 1];
+}
diff --git a/flyteidl2/common/runtime_version.proto b/flyteidl2/common/runtime_version.proto
new file mode 100644
index 00000000000..8196b82f4e4
--- /dev/null
+++ b/flyteidl2/common/runtime_version.proto
@@ -0,0 +1,24 @@
+syntax = "proto3";
+
+package flyteidl2.common;
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/common";
+
+// Runtime information. This is loosely defined to allow for extensibility.
+message RuntimeMetadata {
+ enum RuntimeType {
+ OTHER = 0;
+ FLYTE_SDK = 1;
+ UNION_SDK = 2;
+ }
+
+ // Type of runtime.
+ RuntimeType type = 1;
+
+ // Version of the runtime. All versions should be backward compatible. However, certain cases call for version
+ // checks to ensure tighter validation or setting expectations.
+ string version = 2;
+
+ //+optional It can be used to provide extra information about the runtime (e.g. python, golang... etc.).
+ string flavor = 3;
+}
diff --git a/flyteidl2/connector/connector.proto b/flyteidl2/connector/connector.proto
new file mode 100644
index 00000000000..577038a85cb
--- /dev/null
+++ b/flyteidl2/connector/connector.proto
@@ -0,0 +1,221 @@
+syntax = "proto3";
+
+package flyteidl2.connector;
+
+import "flyteidl2/core/execution.proto";
+import "flyteidl2/core/identifier.proto";
+import "flyteidl2/core/metrics.proto";
+import "flyteidl2/core/security.proto";
+import "flyteidl2/core/tasks.proto";
+import "flyteidl2/task/common.proto";
+import "google/protobuf/duration.proto";
+import "google/protobuf/struct.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/connector";
+
+// Represents a subset of runtime task execution metadata that are relevant to external plugins.
+message TaskExecutionMetadata {
+ // ID of the task execution
+
+ core.TaskExecutionIdentifier task_execution_id = 1;
+ // k8s namespace where the task is executed in
+ string namespace = 2;
+ // Labels attached to the task execution
+ map labels = 3;
+ // Annotations attached to the task execution
+ map annotations = 4;
+ // k8s service account associated with the task execution
+ string k8s_service_account = 5;
+ // Environment variables attached to the task execution
+ map environment_variables = 6;
+ // Represents the maximum number of attempts allowed for a task.
+ // If a task fails, it can be retried up to this maximum number of attempts.
+ int32 max_attempts = 7;
+ // Indicates whether the task execution can be interrupted.
+ // If set to true, the task can be stopped before completion.
+ bool interruptible = 8;
+ // Specifies the threshold for failure count at which the interruptible property
+ // will take effect. If the number of consecutive task failures exceeds this threshold,
+ // interruptible behavior will be activated.
+ int32 interruptible_failure_threshold = 9;
+ // Identity of user running this task execution
+ core.Identity identity = 11;
+}
+
+// Represents a request structure to create task.
+message CreateTaskRequest {
+ // The inputs required to start the execution. All required inputs must be
+ // included in this map. If not required and not provided, defaults apply.
+ // +optional
+ task.Inputs inputs = 1;
+ // Template of the task that encapsulates all the metadata of the task.
+ core.TaskTemplate template = 2;
+ // Prefix for where task output data will be written. (e.g. s3://my-bucket/randomstring)
+ string output_prefix = 3;
+ // subset of runtime task execution metadata.
+ TaskExecutionMetadata task_execution_metadata = 4;
+ // Connection (secret and config) required by the connector.
+ // Connector will use the secret and config in the taskTemplate if it's None.
+ // +optional
+ core.Connection connection = 5;
+}
+
+// Represents a create response structure.
+message CreateTaskResponse {
+ // ResourceMeta is created by the connector. It could be a string (jobId) or a dict (more complex metadata).
+ bytes resource_meta = 1;
+}
+
+message CreateRequestHeader {
+ // Template of the task that encapsulates all the metadata of the task.
+ core.TaskTemplate template = 1;
+ // Prefix for where task output data will be written. (e.g. s3://my-bucket/randomstring)
+ string output_prefix = 2;
+ // subset of runtime task execution metadata.
+ TaskExecutionMetadata task_execution_metadata = 3;
+ // MaxDatasetSizeBytes is the maximum size of the dataset that can be generated by the task.
+ int64 max_dataset_size_bytes = 4;
+ // Connection (secret and config) required by the connector.
+ // Connector will use the secret and config in the taskTemplate if it's None.
+ // +optional
+ core.Connection connection = 5;
+}
+
+// A message used to fetch a job resource from flyte connector server.
+message GetTaskRequest {
+ // Metadata about the resource to be pass to the connector.
+ bytes resource_meta = 2;
+ // A predefined yet extensible Task type identifier.
+ TaskCategory task_category = 3;
+ // Prefix for where task output data will be written. (e.g. s3://my-bucket/randomstring)
+ string output_prefix = 4;
+ // Connection (secret and config) required by the connector.
+ // Connector will use the secret and config in the taskTemplate if it's None.
+ // +optional
+ core.Connection connection = 5;
+}
+
+// Response to get an individual task resource.
+message GetTaskResponse {
+ Resource resource = 1;
+}
+
+message Resource {
+ // The outputs of the execution. It's typically used by sql task. connector service will create a
+ // Structured dataset pointing to the query result table.
+ // +optional
+ task.Outputs outputs = 2;
+ // A descriptive message for the current state. e.g. waiting for cluster.
+ string message = 3;
+ // log information for the task execution.
+ repeated core.TaskLog log_links = 4;
+ // The phase of the execution is used to determine the phase of the plugin's execution.
+ core.TaskExecution.Phase phase = 5;
+ // Custom data specific to the connector.
+ google.protobuf.Struct custom_info = 6;
+}
+
+// A message used to delete a task.
+message DeleteTaskRequest {
+ // Metadata about the resource to be pass to the connector.
+ bytes resource_meta = 2;
+ // A predefined yet extensible Task type identifier.
+ TaskCategory task_category = 3;
+ // Connection (secret and config) required by the connector.
+ // Connector will use the secret and config in the taskTemplate if it's None.
+ // +optional
+ core.Connection connection = 5;
+}
+
+// Response to delete a task.
+message DeleteTaskResponse {}
+
+// A message containing the connector metadata.
+message Connector {
+ // Name is the developer-assigned name of the connector.
+ string name = 1;
+ // Supported_task_categories are the categories of the tasks that the connector can handle.
+ repeated TaskCategory supported_task_categories = 4;
+}
+
+message TaskCategory {
+ // The name of the task type.
+ string name = 1;
+ // The version of the task type.
+ int32 version = 2;
+}
+
+// A request to get an connector.
+message GetConnectorRequest {
+ // The name of the connector.
+ string name = 1;
+}
+
+// A response containing an connector.
+message GetConnectorResponse {
+ Connector connector = 1;
+}
+
+// A request to list all connectors.
+message ListConnectorsRequest {}
+
+// A response containing a list of connectors.
+message ListConnectorsResponse {
+ repeated Connector connectors = 1;
+}
+
+// A request to get the metrics from a task execution.
+message GetTaskMetricsRequest {
+ // Metadata is created by the connector. It could be a string (jobId) or a dict (more complex metadata).
+ bytes resource_meta = 2;
+ // The metrics to query. If empty, will return a default set of metrics.
+ // e.g. EXECUTION_METRIC_USED_CPU_AVG or EXECUTION_METRIC_USED_MEMORY_BYTES_AVG
+ repeated string queries = 3;
+ // Start timestamp, inclusive.
+ google.protobuf.Timestamp start_time = 4;
+ // End timestamp, inclusive..
+ google.protobuf.Timestamp end_time = 5;
+ // Query resolution step width in duration format or float number of seconds.
+ google.protobuf.Duration step = 6;
+ // A predefined yet extensible Task type identifier.
+ TaskCategory task_category = 7;
+}
+
+// A response containing a list of metrics for a task execution.
+message GetTaskMetricsResponse {
+ // The execution metric results.
+ repeated core.ExecutionMetricResult results = 1;
+}
+
+// A request to get the log from a task execution.
+message GetTaskLogsRequest {
+ // Metadata is created by the connector. It could be a string (jobId) or a dict (more complex metadata).
+ bytes resource_meta = 2;
+ // Number of lines to return.
+ uint64 lines = 3;
+ // In the case of multiple pages of results, the server-provided token can be used to fetch the next page
+ // in a query. If there are no more results, this value will be empty.
+ string token = 4;
+ // A predefined yet extensible Task type identifier.
+ TaskCategory task_category = 5;
+}
+
+message GetTaskLogsResponseHeader {
+ // In the case of multiple pages of results, the server-provided token can be used to fetch the next page
+ // in a query. If there are no more results, this value will be empty.
+ string token = 1;
+}
+
+message GetTaskLogsResponseBody {
+ // The execution log results.
+ repeated string results = 1;
+}
+
+// A response containing the logs for a task execution.
+message GetTaskLogsResponse {
+ oneof part {
+ GetTaskLogsResponseHeader header = 1;
+ GetTaskLogsResponseBody body = 2;
+ }
+}
diff --git a/flyteidl2/connector/service.proto b/flyteidl2/connector/service.proto
new file mode 100644
index 00000000000..23d6bb94671
--- /dev/null
+++ b/flyteidl2/connector/service.proto
@@ -0,0 +1,56 @@
+syntax = "proto3";
+package flyteidl2.connector;
+
+import "flyteidl2/connector/connector.proto";
+import "google/api/annotations.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/connector";
+
+// AsyncConnectorService defines an RPC Service that allows executor to send the request to the connector server asynchronously.
+service AsyncConnectorService {
+ // CreateTask sends a task create request to the connector service.
+ rpc CreateTask(flyteidl2.connector.CreateTaskRequest) returns (flyteidl2.connector.CreateTaskResponse) {
+ option (google.api.http) = {
+ post: "/api/v1/connector/task"
+ body: "*"
+ };
+ }
+
+ // Get job status.
+ rpc GetTask(flyteidl2.connector.GetTaskRequest) returns (flyteidl2.connector.GetTaskResponse) {
+ option (google.api.http) = {get: "/api/v1/connector/task/{task_category.name}/{task_category.version}/{resource_meta}"};
+ }
+
+ // Delete the task resource.
+ rpc DeleteTask(flyteidl2.connector.DeleteTaskRequest) returns (flyteidl2.connector.DeleteTaskResponse) {
+ option (google.api.http) = {delete: "/api/v1/connector/task_executions/{task_category.name}/{task_category.version}/{resource_meta}"};
+ }
+
+ // GetTaskMetrics returns one or more task execution metrics, if available.
+ //
+ // Errors include
+ // * OutOfRange if metrics are not available for the specified task time range
+ // * various other errors
+ rpc GetTaskMetrics(flyteidl2.connector.GetTaskMetricsRequest) returns (flyteidl2.connector.GetTaskMetricsResponse) {
+ option (google.api.http) = {get: "/api/v1/connector/task/metrics/{task_category.name}/{task_category.version}/{resource_meta}"};
+ }
+
+ // GetTaskLogs returns task execution logs, if available.
+ rpc GetTaskLogs(flyteidl2.connector.GetTaskLogsRequest) returns (stream flyteidl2.connector.GetTaskLogsResponse) {
+ option (google.api.http) = {get: "/api/v1/connector/task/logs/{task_category.name}/{task_category.version}/{resource_meta}"};
+ }
+}
+
+// ConnectorMetadataService defines an RPC service that is also served over HTTP via grpc-gateway.
+// This service allows executor or users to get the metadata of connectors.
+service ConnectorMetadataService {
+ // Fetch a :ref:`ref_flyteidl2.plugins.Connector` definition.
+ rpc GetConnector(flyteidl2.connector.GetConnectorRequest) returns (flyteidl2.connector.GetConnectorResponse) {
+ option (google.api.http) = {get: "/api/v1/connector/{name}"};
+ }
+
+ // Fetch a list of :ref:`ref_flyteidl2.plugins.Connector` definitions.
+ rpc ListConnectors(flyteidl2.connector.ListConnectorsRequest) returns (flyteidl2.connector.ListConnectorsResponse) {
+ option (google.api.http) = {get: "/api/v1/connectors"};
+ }
+}
diff --git a/flyteidl2/core/artifact_id.proto b/flyteidl2/core/artifact_id.proto
new file mode 100644
index 00000000000..a86707f934e
--- /dev/null
+++ b/flyteidl2/core/artifact_id.proto
@@ -0,0 +1,110 @@
+syntax = "proto3";
+
+package flyteidl2.core;
+
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core";
+
+message ArtifactKey {
+ // Project and domain and suffix needs to be unique across a given artifact store.
+ string project = 1;
+ string domain = 2;
+ string name = 3;
+ string org = 4;
+}
+
+// Only valid for triggers
+message ArtifactBindingData {
+ reserved 1 to 4;
+ // These two fields are only relevant in the partition value case
+ oneof partition_data {
+ string partition_key = 5;
+ bool bind_to_time_partition = 6;
+ }
+
+ // This is only relevant in the time partition case
+ TimeTransform time_transform = 7;
+}
+
+enum Granularity {
+ UNSET = 0;
+ MINUTE = 1;
+ HOUR = 2;
+ DAY = 3; // default
+ MONTH = 4;
+}
+
+enum Operator {
+ MINUS = 0;
+ PLUS = 1;
+}
+
+message TimeTransform {
+ string transform = 1;
+ Operator op = 2;
+}
+
+message InputBindingData {
+ string var = 1;
+}
+
+message RuntimeBinding {}
+
+message LabelValue {
+ oneof value {
+ // The string static value is for use in the Partitions object
+ string static_value = 1;
+
+ // The time value is for use in the TimePartition case
+ google.protobuf.Timestamp time_value = 2;
+ ArtifactBindingData triggered_binding = 3;
+ InputBindingData input_binding = 4;
+ RuntimeBinding runtime_binding = 5;
+ }
+}
+
+message Partitions {
+ map value = 1;
+}
+
+message TimePartition {
+ LabelValue value = 1;
+ Granularity granularity = 2;
+}
+
+message ArtifactID {
+ ArtifactKey artifact_key = 1;
+
+ string version = 2;
+
+ // Think of a partition as a tag on an Artifact, except it's a key-value pair.
+ // Different partitions naturally have different versions (execution ids).
+ Partitions partitions = 3;
+
+ // There is no such thing as an empty time partition - if it's not set, then there is no time partition.
+ TimePartition time_partition = 4;
+}
+
+message ArtifactTag {
+ ArtifactKey artifact_key = 1;
+
+ LabelValue value = 2;
+}
+
+// Uniqueness constraints for Artifacts
+// - project, domain, name, version, partitions
+// Option 2 (tags are standalone, point to an individual artifact id):
+// - project, domain, name, alias (points to one partition if partitioned)
+// - project, domain, name, partition key, partition value
+message ArtifactQuery {
+ oneof identifier {
+ ArtifactID artifact_id = 1;
+ ArtifactTag artifact_tag = 2;
+ string uri = 3;
+
+ // This is used in the trigger case, where a user specifies a value for an input that is one of the triggering
+ // artifacts, or a partition value derived from a triggering artifact.
+ ArtifactBindingData binding = 4;
+ }
+}
diff --git a/flyteidl2/core/catalog.proto b/flyteidl2/core/catalog.proto
new file mode 100644
index 00000000000..f01523c815c
--- /dev/null
+++ b/flyteidl2/core/catalog.proto
@@ -0,0 +1,63 @@
+syntax = "proto3";
+
+package flyteidl2.core;
+
+import "flyteidl2/core/identifier.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core";
+
+// Indicates the status of CatalogCaching. The reason why this is not embedded in TaskNodeMetadata is, that we may use for other types of nodes as well in the future
+enum CatalogCacheStatus {
+ // Used to indicate that caching was disabled
+ CACHE_DISABLED = 0;
+ // Used to indicate that the cache lookup resulted in no matches
+ CACHE_MISS = 1;
+ // used to indicate that the associated artifact was a result of a previous execution
+ CACHE_HIT = 2;
+ // used to indicate that the resultant artifact was added to the cache
+ CACHE_POPULATED = 3;
+ // Used to indicate that cache lookup failed because of an error
+ CACHE_LOOKUP_FAILURE = 4;
+ // Used to indicate that cache lookup failed because of an error
+ CACHE_PUT_FAILURE = 5;
+ // Used to indicate the cache lookup was skipped
+ CACHE_SKIPPED = 6;
+ // Used to indicate that the cache was evicted
+ CACHE_EVICTED = 7;
+}
+
+message CatalogArtifactTag {
+ // Artifact ID is generated name
+ string artifact_id = 1;
+ // Flyte computes the tag automatically, as the hash of the values
+ string name = 2;
+}
+
+// Catalog artifact information with specific metadata
+message CatalogMetadata {
+ // Dataset ID in the catalog
+ Identifier dataset_id = 1;
+ // Artifact tag in the catalog
+ CatalogArtifactTag artifact_tag = 2;
+ // Optional: Source Execution identifier, if this dataset was generated by another execution in Flyte. This is a one-of field and will depend on the caching context
+ oneof source_execution {
+ // Today we only support TaskExecutionIdentifier as a source, as catalog caching only works for task executions
+ TaskExecutionIdentifier source_task_execution = 3;
+ }
+}
+
+message CatalogReservation {
+ // Indicates the status of a catalog reservation operation.
+ enum Status {
+ // Used to indicate that reservations are disabled
+ RESERVATION_DISABLED = 0;
+ // Used to indicate that a reservation was successfully acquired or extended
+ RESERVATION_ACQUIRED = 1;
+ // Used to indicate that an active reservation currently exists
+ RESERVATION_EXISTS = 2;
+ // Used to indicate that the reservation has been successfully released
+ RESERVATION_RELEASED = 3;
+ // Used to indicate that a reservation operation resulted in failure
+ RESERVATION_FAILURE = 4;
+ }
+}
diff --git a/flyteidl2/core/errors.proto b/flyteidl2/core/errors.proto
new file mode 100644
index 00000000000..417f1f6e323
--- /dev/null
+++ b/flyteidl2/core/errors.proto
@@ -0,0 +1,35 @@
+syntax = "proto3";
+
+package flyteidl2.core;
+
+import "flyteidl2/core/execution.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core";
+
+// Error message to propagate detailed errors from container executions to the execution
+// engine.
+message ContainerError {
+ // A simplified code for errors, so that we can provide a glossary of all possible errors.
+ string code = 1;
+ // A detailed error message.
+ string message = 2;
+
+ // Defines a generic error type that dictates the behavior of the retry strategy.
+ enum Kind {
+ NON_RECOVERABLE = 0;
+ RECOVERABLE = 1;
+ }
+
+ // An abstract error kind for this error. Defaults to Non_Recoverable if not specified.
+ Kind kind = 3;
+
+ // Defines the origin of the error (system, user, unknown).
+ ExecutionError.ErrorKind origin = 4;
+}
+
+// Defines the errors.pb file format the container can produce to communicate
+// failure reasons to the execution engine.
+message ErrorDocument {
+ // The error raised during execution.
+ ContainerError error = 1;
+}
diff --git a/flyteidl2/core/execution.proto b/flyteidl2/core/execution.proto
new file mode 100644
index 00000000000..f5c1970f255
--- /dev/null
+++ b/flyteidl2/core/execution.proto
@@ -0,0 +1,166 @@
+syntax = "proto3";
+
+package flyteidl2.core;
+
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core";
+
+// Indicates various phases of Workflow Execution
+message WorkflowExecution {
+ enum Phase {
+ UNDEFINED = 0;
+ QUEUED = 1;
+ RUNNING = 2;
+ SUCCEEDING = 3;
+ SUCCEEDED = 4;
+ FAILING = 5;
+ FAILED = 6;
+ ABORTED = 7;
+ TIMED_OUT = 8;
+ ABORTING = 9;
+ }
+}
+
+// Indicates various phases of Node Execution that only include the time spent to run the nodes/workflows
+message NodeExecution {
+ enum Phase {
+ UNDEFINED = 0;
+ QUEUED = 1;
+ RUNNING = 2;
+ SUCCEEDED = 3;
+ FAILING = 4;
+ FAILED = 5;
+ ABORTED = 6;
+ SKIPPED = 7;
+ TIMED_OUT = 8;
+ DYNAMIC_RUNNING = 9;
+ RECOVERED = 10;
+ }
+}
+
+// Phases that task plugins can go through. Not all phases may be applicable to a specific plugin task,
+// but this is the cumulative list that customers may want to know about for their task.
+message TaskExecution {
+ enum Phase {
+ UNDEFINED = 0;
+ QUEUED = 1;
+ RUNNING = 2;
+ SUCCEEDED = 3;
+ ABORTED = 4;
+ FAILED = 5;
+ // To indicate cases where task is initializing, like: ErrImagePull, ContainerCreating, PodInitializing
+ INITIALIZING = 6;
+ // To address cases, where underlying resource is not available: Backoff error, Resource quota exceeded
+ WAITING_FOR_RESOURCES = 7;
+ RETRYABLE_FAILED = 8;
+ }
+}
+
+// Represents the error message from the execution.
+message ExecutionError {
+ // Error code indicates a grouping of a type of error.
+ // More Info:
+ string code = 1;
+ // Detailed description of the error - including stack trace.
+ string message = 2;
+ // Full error contents accessible via a URI
+ string error_uri = 3;
+ // Error type: System or User
+ enum ErrorKind {
+ UNKNOWN = 0;
+ USER = 1;
+ SYSTEM = 2;
+ }
+ ErrorKind kind = 4;
+ // Timestamp of the error
+ google.protobuf.Timestamp timestamp = 5;
+ // Worker that generated the error
+ string worker = 6;
+}
+
+// Log information for the task that is specific to a log sink
+// When our log story is flushed out, we may have more metadata here like log link expiry
+message TaskLog {
+ enum MessageFormat {
+ UNKNOWN = 0;
+ CSV = 1;
+ JSON = 2;
+ }
+
+ enum LinkType {
+ // The link for task log. For example, the aws cloudwatch logs, gcp stackdriver logs, etc.
+ EXTERNAL = 0;
+ // The link for spark UI, ray dashboard, etc.
+ DASHBOARD = 1;
+ // The link for vscode or other IDEs.
+ IDE = 2;
+ }
+
+ string uri = 1;
+ string name = 2;
+ MessageFormat message_format = 3;
+ google.protobuf.Duration ttl = 4;
+ bool ShowWhilePending = 5;
+ bool HideOnceFinished = 6;
+ LinkType link_type = 7;
+ bool ready = 8;
+ string icon_uri = 9;
+}
+
+// Contains metadata required to identify logs produces by a set of pods
+message LogContext {
+ repeated PodLogContext pods = 1;
+ string primary_pod_name = 2;
+}
+
+// Contains metadata required to identify logs produces by a single pod
+message PodLogContext {
+ string namespace = 1;
+
+ string pod_name = 2;
+
+ repeated ContainerContext containers = 3;
+
+ string primary_container_name = 4;
+
+ repeated ContainerContext init_containers = 5;
+}
+
+// Contains metadata required to identify logs produces by a single container
+message ContainerContext {
+ string container_name = 1;
+
+ // Contains metadata required to identify logs produces by a single light-weight process that was run inside a container
+ message ProcessContext {
+ google.protobuf.Timestamp container_start_time = 1;
+ google.protobuf.Timestamp container_end_time = 2;
+ }
+
+ ProcessContext process = 2;
+}
+
+// Represents customized execution run-time attributes.
+message QualityOfServiceSpec {
+ // Indicates how much queueing delay an execution can tolerate.
+ google.protobuf.Duration queueing_budget = 1;
+
+ // Add future, user-configurable options here
+}
+
+// Indicates the priority of an execution.
+message QualityOfService {
+ enum Tier {
+ // Default: no quality of service specified.
+ UNDEFINED = 0;
+ HIGH = 1;
+ MEDIUM = 2;
+ LOW = 3;
+ }
+
+ oneof designation {
+ Tier tier = 1;
+ QualityOfServiceSpec spec = 2;
+ }
+}
diff --git a/flyteidl2/core/identifier.proto b/flyteidl2/core/identifier.proto
new file mode 100644
index 00000000000..522c01eb608
--- /dev/null
+++ b/flyteidl2/core/identifier.proto
@@ -0,0 +1,80 @@
+syntax = "proto3";
+
+package flyteidl2.core;
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core";
+
+// Indicates a resource type within Flyte.
+enum ResourceType {
+ UNSPECIFIED = 0;
+ TASK = 1;
+ WORKFLOW = 2;
+ LAUNCH_PLAN = 3;
+ // A dataset represents an entity modeled in Flyte DataCatalog. A Dataset is also a versioned entity and can be a compilation of multiple individual objects.
+ // Eventually all Catalog objects should be modeled similar to Flyte Objects. The Dataset entities makes it possible for the UI and CLI to act on the objects
+ // in a similar manner to other Flyte objects
+ DATASET = 4;
+}
+
+// Encapsulation of fields that uniquely identifies a Flyte resource.
+message Identifier {
+ // Identifies the specific type of resource that this identifier corresponds to.
+ core.ResourceType resource_type = 1;
+
+ // Name of the project the resource belongs to.
+ string project = 2;
+
+ // Name of the domain the resource belongs to.
+ // A domain can be considered as a subset within a specific project.
+ string domain = 3;
+
+ // User provided value for the resource.
+ string name = 4;
+
+ // Specific version of the resource.
+ string version = 5;
+
+ // Optional, org key applied to the resource.
+ string org = 6;
+}
+
+// Encapsulation of fields that uniquely identifies a Flyte workflow execution
+message WorkflowExecutionIdentifier {
+ // Name of the project the resource belongs to.
+ string project = 1;
+
+ // Name of the domain the resource belongs to.
+ // A domain can be considered as a subset within a specific project.
+ string domain = 2;
+
+ // User or system provided value for the resource.
+ string name = 4;
+
+ // Optional, org key applied to the resource.
+ string org = 5;
+}
+
+// Encapsulation of fields that identify a Flyte node execution entity.
+message NodeExecutionIdentifier {
+ string node_id = 1;
+
+ WorkflowExecutionIdentifier execution_id = 2;
+}
+
+// Encapsulation of fields that identify a Flyte task execution entity.
+message TaskExecutionIdentifier {
+ core.Identifier task_id = 1;
+
+ core.NodeExecutionIdentifier node_execution_id = 2;
+
+ uint32 retry_attempt = 3;
+}
+
+// Encapsulation of fields the uniquely identify a signal.
+message SignalIdentifier {
+ // Unique identifier for a signal.
+ string signal_id = 1;
+
+ // Identifies the Flyte workflow execution this signal belongs to.
+ WorkflowExecutionIdentifier execution_id = 2;
+}
diff --git a/flyteidl2/core/interface.proto b/flyteidl2/core/interface.proto
new file mode 100644
index 00000000000..9506d615092
--- /dev/null
+++ b/flyteidl2/core/interface.proto
@@ -0,0 +1,70 @@
+syntax = "proto3";
+
+package flyteidl2.core;
+
+import "flyteidl2/core/artifact_id.proto";
+import "flyteidl2/core/literals.proto";
+import "flyteidl2/core/types.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core";
+
+// Defines a strongly typed variable.
+message Variable {
+ // Variable literal type.
+ LiteralType type = 1;
+
+ //+optional string describing input variable
+ string description = 2;
+
+ //+optional This object allows the user to specify how Artifacts are created.
+ // name, tag, partitions can be specified. The other fields (version and project/domain) are ignored.
+ core.ArtifactID artifact_partial_id = 3;
+
+ core.ArtifactTag artifact_tag = 4;
+}
+
+// Defines a single entity for variable.
+message VariableEntry {
+ string key = 1;
+ Variable value = 2;
+}
+
+// A map of Variables
+message VariableMap {
+ // Use repeated key value pair to maintain ordering across different languages.
+ repeated VariableEntry variables = 1;
+}
+
+// Defines strongly typed inputs and outputs.
+message TypedInterface {
+ VariableMap inputs = 1;
+ VariableMap outputs = 2;
+}
+
+// A parameter is used as input to a launch plan and has
+// the special ability to have a default value or mark itself as required.
+message Parameter {
+ //+required Variable. Defines the type of the variable backing this parameter.
+ Variable var = 1;
+
+ //+optional
+ oneof behavior {
+ // Defines a default value that has to match the variable type defined.
+ Literal default = 2;
+
+ //+optional, is this value required to be filled.
+ bool required = 3;
+
+ // This is an execution time search basically that should result in exactly one Artifact with a Type that
+ // matches the type of the variable.
+ core.ArtifactQuery artifact_query = 4;
+
+ core.ArtifactID artifact_id = 5;
+ }
+}
+
+// A map of Parameters.
+message ParameterMap {
+ // Defines a map of parameter names to parameters.
+ map parameters = 1;
+}
diff --git a/flyteidl2/core/literals.proto b/flyteidl2/core/literals.proto
new file mode 100644
index 00000000000..a8fa64ebb8e
--- /dev/null
+++ b/flyteidl2/core/literals.proto
@@ -0,0 +1,204 @@
+syntax = "proto3";
+
+package flyteidl2.core;
+
+import "flyteidl2/core/types.proto";
+import "google/protobuf/duration.proto";
+import "google/protobuf/struct.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core";
+
+// Primitive Types
+message Primitive {
+ // Defines one of simple primitive types. These types will get translated into different programming languages as
+ // described in https://developers.google.com/protocol-buffers/docs/proto#scalar.
+ oneof value {
+ int64 integer = 1;
+ double float_value = 2;
+ string string_value = 3;
+ bool boolean = 4;
+ google.protobuf.Timestamp datetime = 5;
+ google.protobuf.Duration duration = 6;
+ }
+}
+
+// Used to denote a nil/null/None assignment to a scalar value. The underlying LiteralType for Void is intentionally
+// undefined since it can be assigned to a scalar of any LiteralType.
+message Void {}
+
+// Refers to an offloaded set of files. It encapsulates the type of the store and a unique uri for where the data is.
+// There are no restrictions on how the uri is formatted since it will depend on how to interact with the store.
+message Blob {
+ BlobMetadata metadata = 1;
+ string uri = 3;
+}
+
+message BlobMetadata {
+ BlobType type = 1;
+}
+
+// A simple byte array with a tag to help different parts of the system communicate about what is in the byte array.
+// It's strongly advisable that consumers of this type define a unique tag and validate the tag before parsing the data.
+message Binary {
+ bytes value = 1; // Serialized data (MessagePack) for supported types like Dataclass, Pydantic BaseModel, and untyped dict.
+ string tag = 2; // The serialization format identifier (e.g., MessagePack). Consumers must define unique tags and validate them before deserialization.
+}
+
+// A strongly typed schema that defines the interface of data retrieved from the underlying storage medium.
+message Schema {
+ string uri = 1;
+ SchemaType type = 3;
+}
+
+// The runtime representation of a tagged union value. See `UnionType` for more details.
+message Union {
+ Literal value = 1;
+ LiteralType type = 2;
+}
+
+message StructuredDatasetMetadata {
+ // Bundle the type information along with the literal.
+ // This is here because StructuredDatasets can often be more defined at run time than at compile time.
+ // That is, at compile time you might only declare a task to return a pandas dataframe or a StructuredDataset,
+ // without any column information, but at run time, you might have that column information.
+ // flytekit python will copy this type information into the literal, from the type information, if not provided by
+ // the various plugins (encoders).
+ // Since this field is run time generated, it's not used for any type checking.
+ StructuredDatasetType structured_dataset_type = 1;
+}
+
+message StructuredDataset {
+ // String location uniquely identifying where the data is.
+ // Should start with the storage location (e.g. s3://, gs://, bq://, etc.)
+ string uri = 1;
+
+ StructuredDatasetMetadata metadata = 2;
+}
+
+message Scalar {
+ oneof value {
+ Primitive primitive = 1;
+ Blob blob = 2;
+ Binary binary = 3;
+ Schema schema = 4;
+ Void none_type = 5;
+ Error error = 6;
+ google.protobuf.Struct generic = 7;
+ StructuredDataset structured_dataset = 8;
+ Union union = 9;
+ }
+}
+
+// A simple value. This supports any level of nesting (e.g. array of array of array of Blobs) as well as simple primitives.
+message Literal {
+ reserved 6, 7;
+ oneof value {
+ // A simple value.
+ Scalar scalar = 1;
+
+ // A collection of literals to allow nesting.
+ LiteralCollection collection = 2;
+
+ // A map of strings to literals.
+ LiteralMap map = 3;
+
+ // Offloaded literal metadata
+ // When you deserialize the offloaded metadata, it would be of Literal and its type would be defined by LiteralType stored in offloaded_metadata.
+ LiteralOffloadedMetadata offloaded_metadata = 8;
+ }
+
+ // A hash representing this literal.
+ // This is used for caching purposes. For more details refer to RFC 1893
+ // (https://github.com/flyteorg/flyte/blob/master/rfc/system/1893-caching-of-offloaded-objects.md)
+ string hash = 4;
+
+ // Additional metadata for literals.
+ map metadata = 5;
+}
+
+// A message that contains the metadata of the offloaded data.
+message LiteralOffloadedMetadata {
+ // The location of the offloaded core.Literal.
+ string uri = 1;
+
+ // The size of the offloaded data.
+ uint64 size_bytes = 2;
+
+ // The inferred literal type of the offloaded data.
+ LiteralType inferred_type = 3;
+}
+
+// A collection of literals. This is a workaround since oneofs in proto messages cannot contain a repeated field.
+message LiteralCollection {
+ repeated Literal literals = 1;
+}
+
+// A map of literals. This is a workaround since oneofs in proto messages cannot contain a repeated field.
+message LiteralMap {
+ map literals = 1;
+}
+
+// A collection of BindingData items.
+message BindingDataCollection {
+ repeated BindingData bindings = 1;
+}
+
+// A map of BindingData items.
+message BindingDataMap {
+ map bindings = 1;
+}
+
+message UnionInfo {
+ LiteralType targetType = 1;
+}
+
+// Specifies either a simple value or a reference to another output.
+message BindingData {
+ oneof value {
+ // A simple scalar value.
+ Scalar scalar = 1;
+
+ // A collection of binding data. This allows nesting of binding data to any number
+ // of levels.
+ BindingDataCollection collection = 2;
+
+ // References an output promised by another node.
+ OutputReference promise = 3;
+
+ // A map of bindings. The key is always a string.
+ BindingDataMap map = 4;
+
+ // Offloaded literal metadata
+ // When you deserialize the offloaded metadata, it would be of Literal and its type would be defined by LiteralType stored in offloaded_metadata.
+ // Used for nodes that don't have promises from upstream nodes such as ArrayNode subNodes.
+ LiteralOffloadedMetadata offloaded_metadata = 6;
+ }
+
+ UnionInfo union = 5;
+}
+
+// An input/output binding of a variable to either static value or a node output.
+message Binding {
+ // Variable name must match an input/output variable of the node.
+ string var = 1;
+
+ // Data to use to bind this variable.
+ BindingData binding = 2;
+}
+
+// A generic key value pair.
+message KeyValuePair {
+ //required.
+ string key = 1;
+
+ //+optional.
+ string value = 2;
+}
+
+// Retry strategy associated with an executable unit.
+message RetryStrategy {
+ // Number of retries. Retries will be consumed when the job fails with a recoverable error.
+ // The number of retries must be less than or equals to 10.
+ uint32 retries = 5;
+}
diff --git a/flyteidl2/core/metrics.proto b/flyteidl2/core/metrics.proto
new file mode 100644
index 00000000000..30c270e7319
--- /dev/null
+++ b/flyteidl2/core/metrics.proto
@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+package flyteidl2.core;
+
+import "google/protobuf/struct.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core";
+
+// ExecutionMetrics is a collection of metrics that are collected during the execution of a Flyte task.
+message ExecutionMetricResult {
+ // The metric this data represents. e.g. EXECUTION_METRIC_USED_CPU_AVG or EXECUTION_METRIC_USED_MEMORY_BYTES_AVG.
+ string metric = 1;
+
+ // The result data in prometheus range query result format
+ // https://prometheus.io/docs/prometheus/latest/querying/api/#expression-query-result-formats.
+ // This may include multiple time series, differentiated by their metric labels.
+ // Start time is greater of (execution attempt start, 48h ago)
+ // End time is lesser of (execution attempt end, now)
+ google.protobuf.Struct data = 2;
+}
diff --git a/flyteidl2/core/security.proto b/flyteidl2/core/security.proto
new file mode 100644
index 00000000000..e89f8c8f606
--- /dev/null
+++ b/flyteidl2/core/security.proto
@@ -0,0 +1,148 @@
+syntax = "proto3";
+
+package flyteidl2.core;
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core";
+
+// Secret encapsulates information about the secret a task needs to proceed. An environment variable
+// FLYTE_SECRETS_ENV_PREFIX will be passed to indicate the prefix of the environment variables that will be present if
+// secrets are passed through environment variables.
+// FLYTE_SECRETS_DEFAULT_DIR will be passed to indicate the prefix of the path where secrets will be mounted if secrets
+// are passed through file mounts.
+message Secret {
+ enum MountType {
+ // Default case, indicates the client can tolerate either mounting options.
+ ANY = 0;
+
+ // ENV_VAR indicates the secret needs to be mounted as an environment variable.
+ ENV_VAR = 1;
+
+ // FILE indicates the secret needs to be mounted as a file.
+ FILE = 2;
+ }
+
+ // The name of the secret group where to find the key referenced below. For K8s secrets, this should be the name of
+ // the v1/secret object. For Confidant, this should be the Credential name. For Vault, this should be the secret name.
+ // For AWS Secret Manager, this should be the name of the secret.
+ // +required
+ string group = 1;
+
+ // The group version to fetch. This is not supported in all secret management systems. It'll be ignored for the ones
+ // that do not support it.
+ // +optional
+ string group_version = 2;
+
+ // The name of the secret to mount. This has to match an existing secret in the system. It's up to the implementation
+ // of the secret management system to require case sensitivity. For K8s secrets, Confidant and Vault, this should
+ // match one of the keys inside the secret. For AWS Secret Manager, it's ignored.
+ // +optional
+ string key = 3;
+
+ // mount_requirement is optional. Indicates where the secret has to be mounted. If provided, the execution will fail
+ // if the underlying key management system cannot satisfy that requirement. If not provided, the default location
+ // will depend on the key management system.
+ // +optional
+ MountType mount_requirement = 4;
+
+ // env_var is optional. Custom environment variable to set the value of the secret. If mount_requirement is ENV_VAR,
+ // then the value is the secret itself. If mount_requirement is FILE, then the value is the path to the secret file.
+ // +optional
+ string env_var = 5;
+}
+
+message Connection {
+ // The task type that the connection is used for.
+ string task_type = 1;
+ // The credentials to use for the connection, such as API keys, OAuth2 tokens, etc.
+ // The key is the name of the secret, and it's defined in the flytekit.
+ // flytekit uses the key to locate the desired secret within the map.
+ map secrets = 2;
+
+ // The configuration to use for the connection, such as the endpoint, account name, etc.
+ // The key is the name of the config, and it's defined in the flytekit.
+ map configs = 3;
+}
+
+// OAuth2Client encapsulates OAuth2 Client Credentials to be used when making calls on behalf of that task.
+message OAuth2Client {
+ // client_id is the public id for the client to use. The system will not perform any pre-auth validation that the
+ // secret requested matches the client_id indicated here.
+ // +required
+ string client_id = 1;
+
+ // client_secret is a reference to the secret used to authenticate the OAuth2 client.
+ // +required
+ Secret client_secret = 2;
+}
+
+// Identity encapsulates the various security identities a task can run as. It's up to the underlying plugin to pick the
+// right identity for the execution environment.
+message Identity {
+ // iam_role references the fully qualified name of Identity & Access Management role to impersonate.
+ string iam_role = 1;
+
+ // k8s_service_account references a kubernetes service account to impersonate.
+ string k8s_service_account = 2;
+
+ // oauth2_client references an oauth2 client. Backend plugins can use this information to impersonate the client when
+ // making external calls.
+ OAuth2Client oauth2_client = 3;
+
+ // execution_identity references the subject who makes the execution
+ string execution_identity = 4;
+}
+
+// OAuth2TokenRequest encapsulates information needed to request an OAuth2 token.
+// FLYTE_TOKENS_ENV_PREFIX will be passed to indicate the prefix of the environment variables that will be present if
+// tokens are passed through environment variables.
+// FLYTE_TOKENS_PATH_PREFIX will be passed to indicate the prefix of the path where secrets will be mounted if tokens
+// are passed through file mounts.
+message OAuth2TokenRequest {
+ // Type of the token requested.
+ enum Type {
+ // CLIENT_CREDENTIALS indicates a 2-legged OAuth token requested using client credentials.
+ CLIENT_CREDENTIALS = 0;
+ }
+
+ // name indicates a unique id for the token request within this task token requests. It'll be used as a suffix for
+ // environment variables and as a filename for mounting tokens as files.
+ // +required
+ string name = 1;
+
+ // type indicates the type of the request to make. Defaults to CLIENT_CREDENTIALS.
+ // +required
+ Type type = 2;
+
+ // client references the client_id/secret to use to request the OAuth2 token.
+ // +required
+ OAuth2Client client = 3;
+
+ // idp_discovery_endpoint references the discovery endpoint used to retrieve token endpoint and other related
+ // information.
+ // +optional
+ string idp_discovery_endpoint = 4;
+
+ // token_endpoint references the token issuance endpoint. If idp_discovery_endpoint is not provided, this parameter is
+ // mandatory.
+ // +optional
+ string token_endpoint = 5;
+}
+
+// SecurityContext holds security attributes that apply to tasks.
+message SecurityContext {
+ // run_as encapsulates the identity a pod should run as. If the task fills in multiple fields here, it'll be up to the
+ // backend plugin to choose the appropriate identity for the execution engine the task will run on.
+ Identity run_as = 1;
+
+ // secrets indicate the list of secrets the task needs in order to proceed. Secrets will be mounted/passed to the
+ // pod as it starts. If the plugin responsible for kicking of the task will not run it on a flyte cluster (e.g. AWS
+ // Batch), it's the responsibility of the plugin to fetch the secret (which means propeller identity will need access
+ // to the secret) and to pass it to the remote execution engine.
+ repeated Secret secrets = 2;
+
+ // tokens indicate the list of token requests the task needs in order to proceed. Tokens will be mounted/passed to the
+ // pod as it starts. If the plugin responsible for kicking of the task will not run it on a flyte cluster (e.g. AWS
+ // Batch), it's the responsibility of the plugin to fetch the secret (which means propeller identity will need access
+ // to the secret) and to pass it to the remote execution engine.
+ repeated OAuth2TokenRequest tokens = 3;
+}
diff --git a/flyteidl2/core/tasks.proto b/flyteidl2/core/tasks.proto
new file mode 100644
index 00000000000..11bf2ccb176
--- /dev/null
+++ b/flyteidl2/core/tasks.proto
@@ -0,0 +1,423 @@
+syntax = "proto3";
+
+package flyteidl2.core;
+
+import "flyteidl2/common/identifier.proto";
+import "flyteidl2/core/execution.proto";
+import "flyteidl2/core/identifier.proto";
+import "flyteidl2/core/interface.proto";
+import "flyteidl2/core/literals.proto";
+import "flyteidl2/core/security.proto";
+import "google/protobuf/duration.proto";
+import "google/protobuf/struct.proto";
+import "google/protobuf/wrappers.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core";
+
+// A customizable interface to convey resources requested for a container. This can be interpreted differently for different
+// container engines.
+message Resources {
+ // Known resource names.
+ enum ResourceName {
+ UNKNOWN = 0;
+ CPU = 1;
+ GPU = 2;
+ MEMORY = 3;
+ STORAGE = 4;
+ // For Kubernetes-based deployments, pods use ephemeral local storage for scratch space, caching, and for logs.
+ EPHEMERAL_STORAGE = 5;
+ }
+
+ // Encapsulates a resource name and value.
+ message ResourceEntry {
+ // Resource name.
+ ResourceName name = 1;
+
+ // Value must be a valid k8s quantity. See
+ // https://github.com/kubernetes/apimachinery/blob/master/pkg/api/resource/quantity.go#L30-L80
+ string value = 2;
+ }
+
+ // The desired set of resources requested. ResourceNames must be unique within the list.
+ repeated ResourceEntry requests = 1;
+
+ // Defines a set of bounds (e.g. min/max) within which the task can reliably run. ResourceNames must be unique
+ // within the list.
+ repeated ResourceEntry limits = 2;
+}
+
+// Metadata associated with the GPU accelerator to allocate to a task. Contains
+// information about device type, and for multi-instance GPUs, the partition size to
+// use.
+message GPUAccelerator {
+ // Specifies the class of accelerator device.
+ enum DeviceClass {
+ // NVIDIA GPU devices (default for backward compatibility)
+ NVIDIA_GPU = 0;
+ // Google TPU devices
+ GOOGLE_TPU = 1;
+ // Amazon Neuron devices
+ AMAZON_NEURON = 2;
+ // AMD GPU devices
+ AMD_GPU = 3;
+ // Habana Gaudi devices
+ HABANA_GAUDI = 4;
+ }
+
+ // This can be any arbitrary string, and should be informed by the labels or taints
+ // associated with the nodes in question. Default cloud provider labels typically
+ // use the following values: `nvidia-tesla-t4`, `nvidia-tesla-a100`, etc.
+ string device = 1;
+ oneof partition_size_value {
+ bool unpartitioned = 2;
+ // Like `device`, this can be any arbitrary string, and should be informed by
+ // the labels or taints associated with the nodes in question. Default cloud
+ // provider labels typically use the following values: `1g.5gb`, `2g.10gb`, etc.
+ string partition_size = 3;
+ }
+ // The class of accelerator device. Defaults to NVIDIA_GPU if not specified.
+ DeviceClass device_class = 4;
+}
+
+// Metadata associated with configuring a shared memory volume for a task.
+message SharedMemory {
+ // Mount path to place in container
+ string mount_path = 1;
+ // Name for volume
+ string mount_name = 2;
+ // Size limit for shared memory. If not set, then the shared memory is equal
+ // to the allocated memory.
+ // +optional
+ string size_limit = 3;
+}
+
+// Encapsulates all non-standard resources, not captured by v1.ResourceRequirements, to
+// allocate to a task.
+message ExtendedResources {
+ // GPU accelerator to select for task. Contains information about device type, and
+ // for multi-instance GPUs, the partition size to use.
+ GPUAccelerator gpu_accelerator = 1;
+ SharedMemory shared_memory = 2;
+}
+
+// Runtime information. This is loosely defined to allow for extensibility.
+message RuntimeMetadata {
+ enum RuntimeType {
+ OTHER = 0;
+ FLYTE_SDK = 1;
+ }
+
+ // Type of runtime.
+ RuntimeType type = 1;
+
+ // Version of the runtime. All versions should be backward compatible. However, certain cases call for version
+ // checks to ensure tighter validation or setting expectations.
+ string version = 2;
+
+ //+optional It can be used to provide extra information about the runtime (e.g. python, golang... etc.).
+ string flavor = 3;
+}
+
+// Task Metadata
+message TaskMetadata {
+ // Field number 10 is reserved because we are reusing the name generates_deck for field number 15,
+ // but with a different type.
+ reserved 10;
+
+ // Indicates whether the system should attempt to lookup this task's output to avoid duplication of work.
+ bool discoverable = 1;
+
+ // Runtime information about the task.
+ RuntimeMetadata runtime = 2;
+
+ // The overall timeout of a task including user-triggered retries.
+ google.protobuf.Duration timeout = 4;
+
+ // Number of retries per task.
+ RetryStrategy retries = 5;
+
+ // Indicates a logical version to apply to this task for the purpose of discovery.
+ string discovery_version = 6;
+
+ // If set, this indicates that this task is deprecated. This will enable owners of tasks to notify consumers
+ // of the ending of support for a given task.
+ string deprecated_error_message = 7;
+
+ // For interruptible we will populate it at the node level but require it be part of TaskMetadata
+ // for a user to set the value.
+ // We are using oneof instead of bool because otherwise we would be unable to distinguish between value being
+ // set by the user or defaulting to false.
+ // The logic of handling precedence will be done as part of flytepropeller.
+
+ // Identify whether task is interruptible
+ oneof interruptible_value {
+ bool interruptible = 8;
+ }
+
+ // Indicates whether the system should attempt to execute discoverable instances in serial to avoid duplicate work
+ bool cache_serializable = 9;
+
+ // Arbitrary tags that allow users and the platform to store small but arbitrary labels
+ map tags = 11;
+
+ // pod_template_name is the unique name of a PodTemplate k8s resource to be used as the base configuration if this
+ // task creates a k8s Pod. If this value is set, the specified PodTemplate will be used instead of, but applied
+ // identically as, the default PodTemplate configured in FlytePropeller.
+ string pod_template_name = 12;
+
+ // cache_ignore_input_vars is the input variables that should not be included when calculating hash for cache.
+ repeated string cache_ignore_input_vars = 13;
+ // is_eager indicates whether the task is eager or not.
+ // This would be used by CreateTask endpoint.
+ bool is_eager = 14;
+
+ // Indicates whether the task will generate a deck when it finishes executing.
+ // The BoolValue can have three states:
+ // - nil: The value is not set.
+ // - true: The task will generate a deck.
+ // - false: The task will not generate a deck.
+ google.protobuf.BoolValue generates_deck = 15;
+
+ // Metadata applied to task pods or task CR objects.
+ // In flytekit, labels and annotations resulting in this metadata field
+ // are provided via `@task(labels=..., annotations=...)`.
+ // For tasks backed by pods like PythonFunctionTask, these take precedence
+ // over the metadata provided via `@task(pod_template=PodTemplate(labels=...))` which are transported
+ // in the K8sPod message. For tasks backed by CRDs, this metadata is applied to
+ // the CR object itself while the metadata in the pod template/K8sPod is applied
+ // to the pod template spec of the CR object.
+ K8sObjectMetadata metadata = 16;
+
+ // Whether the task is able to run the debugger (vscode server) inside the task container.
+ bool debuggable = 17;
+ // Log links associated with this task.
+ repeated core.TaskLog log_links = 18;
+
+ // RunIdentifier of the remote image build run that produced the container image for this task.
+ common.RunIdentifier image_build_run = 19;
+
+ // IsEntryPoint indicates whether this task will launch additional actions as it runs.
+ bool is_entrypoint = 20;
+
+ // Remote URI pointing to the code bundle used to build this task.
+ string code_bundle_uri = 21;
+}
+
+// A Task structure that uniquely identifies a task in the system
+// Tasks are registered as a first step in the system.
+message TaskTemplate {
+ // Auto generated taskId by the system. Task Id uniquely identifies this task globally.
+ Identifier id = 1;
+
+ // A predefined yet extensible Task type identifier. This can be used to customize any of the components. If no
+ // extensions are provided in the system, Flyte will resolve the this task to its TaskCategory and default the
+ // implementation registered for the TaskCategory.
+ string type = 2;
+
+ // Extra metadata about the task.
+ TaskMetadata metadata = 3;
+
+ // A strongly typed interface for the task. This enables others to use this task within a workflow and guarantees
+ // compile-time validation of the workflow to avoid costly runtime failures.
+ TypedInterface interface = 4;
+
+ // Custom data about the task. This is extensible to allow various plugins in the system.
+ google.protobuf.Struct custom = 5;
+
+ // Known target types that the system will guarantee plugins for. Custom SDK plugins are allowed to set these if needed.
+ // If no corresponding execution-layer plugins are found, the system will default to handling these using built-in
+ // handlers.
+ oneof target {
+ Container container = 6;
+ K8sPod k8s_pod = 17;
+ Sql sql = 18;
+ }
+
+ // This can be used to customize task handling at execution time for the same task type.
+ int32 task_type_version = 7;
+
+ // security_context encapsulates security attributes requested to run this task.
+ SecurityContext security_context = 8;
+
+ // Encapsulates all non-standard resources, not captured by
+ // v1.ResourceRequirements, to allocate to a task.
+ ExtendedResources extended_resources = 9;
+
+ // Metadata about the custom defined for this task. This is extensible to allow various plugins in the system
+ // to use as required.
+ // reserve the field numbers 1 through 15 for very frequently occurring message elements
+ map config = 16;
+}
+
+// ----------------- First class Plugins
+
+// Defines port properties for a container.
+message ContainerPort {
+ // Number of port to expose on the pod's IP address.
+ // This must be a valid port number, 0 < x < 65536.
+ uint32 container_port = 1;
+ // Name of the port to expose on the pod's IP address.
+ string name = 2;
+}
+
+message Container {
+ // Container image url. Eg: docker/redis:latest
+ string image = 1;
+
+ // Command to be executed, if not provided, the default entrypoint in the container image will be used.
+ repeated string command = 2;
+
+ // These will default to Flyte given paths. If provided, the system will not append known paths. If the task still
+ // needs flyte's inputs and outputs path, add $(FLYTE_INPUT_FILE), $(FLYTE_OUTPUT_FILE) wherever makes sense and the
+ // system will populate these before executing the container.
+ repeated string args = 3;
+
+ // Container resources requirement as specified by the container engine.
+ Resources resources = 4;
+
+ // Environment variables will be set as the container is starting up.
+ repeated KeyValuePair env = 5;
+
+ // Allows extra configs to be available for the container.
+ // TODO: elaborate on how configs will become available.
+ // Deprecated, please use TaskTemplate.config instead.
+ repeated KeyValuePair config = 6 [deprecated = true];
+
+ // Ports to open in the container. This feature is not supported by all execution engines. (e.g. supported on K8s but
+ // not supported on AWS Batch)
+ // Only K8s
+ repeated ContainerPort ports = 7;
+
+ // BETA: Optional configuration for DataLoading. If not specified, then default values are used.
+ // This makes it possible to to run a completely portable container, that uses inputs and outputs
+ // only from the local file-system and without having any reference to flyteidl. This is supported only on K8s at the moment.
+ // If data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories
+ // are not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation
+ // to understand the default paths.
+ // Only K8s
+ DataLoadingConfig data_config = 9;
+
+ // Architecture-type the container image supports.
+ enum Architecture {
+ UNKNOWN = 0;
+ AMD64 = 1;
+ ARM64 = 2;
+ ARM_V6 = 3;
+ ARM_V7 = 4;
+ }
+ Architecture architecture = 10;
+}
+
+// Strategy to use when dealing with Blob, Schema, or multipart blob data (large datasets)
+message IOStrategy {
+ // Mode to use for downloading
+ enum DownloadMode {
+ // All data will be downloaded before the main container is executed
+ DOWNLOAD_EAGER = 0;
+ // Data will be downloaded as a stream and an End-Of-Stream marker will be written to indicate all data has been downloaded. Refer to protocol for details
+ DOWNLOAD_STREAM = 1;
+ // Large objects (offloaded) will not be downloaded
+ DO_NOT_DOWNLOAD = 2;
+ }
+ // Mode to use for uploading
+ enum UploadMode {
+ // All data will be uploaded after the main container exits
+ UPLOAD_ON_EXIT = 0;
+ // Data will be uploaded as it appears. Refer to protocol specification for details
+ UPLOAD_EAGER = 1;
+ // Data will not be uploaded, only references will be written
+ DO_NOT_UPLOAD = 2;
+ }
+ // Mode to use to manage downloads
+ DownloadMode download_mode = 1;
+ // Mode to use to manage uploads
+ UploadMode upload_mode = 2;
+}
+
+// This configuration allows executing raw containers in Flyte using the Flyte CoPilot system.
+// Flyte CoPilot, eliminates the needs of flytekit or sdk inside the container. Any inputs required by the users container are side-loaded in the input_path
+// Any outputs generated by the user container - within output_path are automatically uploaded.
+message DataLoadingConfig {
+ // LiteralMapFormat decides the encoding format in which the input metadata should be made available to the containers.
+ // If the user has access to the protocol buffer definitions, it is recommended to use the PROTO format.
+ // JSON and YAML do not need any protobuf definitions to read it
+ // All remote references in core.LiteralMap are replaced with local filesystem references (the data is downloaded to local filesystem)
+ enum LiteralMapFormat {
+ // JSON / YAML for the metadata (which contains inlined primitive values). The representation is inline with the standard json specification as specified - https://www.json.org/json-en.html
+ JSON = 0;
+ YAML = 1;
+ // Proto is a serialized binary of `core.LiteralMap` defined in flyteidl/core
+ PROTO = 2;
+ }
+ // Flag enables DataLoading Config. If this is not set, data loading will not be used!
+ bool enabled = 1;
+ // File system path (start at root). This folder will contain all the inputs exploded to a separate file.
+ // Example, if the input interface needs (x: int, y: blob, z: multipart_blob) and the input path is '/var/flyte/inputs', then the file system will look like
+ // /var/flyte/inputs/inputs. .pb .json .yaml> -> Format as defined previously. The Blob and Multipart blob will reference local filesystem instead of remote locations
+ // /var/flyte/inputs/x -> X is a file that contains the value of x (integer) in string format
+ // /var/flyte/inputs/y -> Y is a file in Binary format
+ // /var/flyte/inputs/z/... -> Note Z itself is a directory
+ // More information about the protocol - refer to docs #TODO reference docs here
+ string input_path = 2;
+ // File system path (start at root). This folder should contain all the outputs for the task as individual files and/or an error text file
+ string output_path = 3;
+ // In the inputs folder, there will be an additional summary/metadata file that contains references to all files or inlined primitive values.
+ // This format decides the actual encoding for the data. Refer to the encoding to understand the specifics of the contents and the encoding
+ LiteralMapFormat format = 4;
+ IOStrategy io_strategy = 5;
+}
+
+// Defines a pod spec and additional pod metadata that is created when a task is executed.
+message K8sPod {
+ // Contains additional metadata for building a kubernetes pod.
+ K8sObjectMetadata metadata = 1;
+
+ // Defines the primary pod spec created when a task is executed.
+ // This should be a JSON-marshalled pod spec, which can be defined in
+ // - go, using: https://github.com/kubernetes/api/blob/release-1.21/core/v1/types.go#L2936
+ // - python: using https://github.com/kubernetes-client/python/blob/release-19.0/kubernetes/client/models/v1_pod_spec.py
+ google.protobuf.Struct pod_spec = 2;
+
+ // BETA: Optional configuration for DataLoading. If not specified, then default values are used.
+ // This makes it possible to to run a completely portable container, that uses inputs and outputs
+ // only from the local file-system and without having any reference to flytekit. This is supported only on K8s at the moment.
+ // If data loading is enabled, then data will be mounted in accompanying directories specified in the DataLoadingConfig. If the directories
+ // are not specified, inputs will be mounted onto and outputs will be uploaded from a pre-determined file-system path. Refer to the documentation
+ // to understand the default paths.
+ // Only K8s
+ DataLoadingConfig data_config = 3;
+
+ // Defines the primary container name when pod template override is executed.
+ string primary_container_name = 4;
+}
+
+// Metadata for building a kubernetes object when a task is executed.
+message K8sObjectMetadata {
+ // Optional labels to add to the pod definition.
+ map labels = 1;
+
+ // Optional annotations to add to the pod definition.
+ map annotations = 2;
+}
+
+// Sql represents a generic sql workload with a statement and dialect.
+message Sql {
+ // The actual query to run, the query can have templated parameters.
+ // We use Flyte's Golang templating format for Query templating.
+ // For example,
+ // insert overwrite directory '{{ .rawOutputDataPrefix }}' stored as parquet
+ // select *
+ // from my_table
+ // where ds = '{{ .Inputs.ds }}'
+ string statement = 1;
+ // The dialect of the SQL statement. This is used to validate and parse SQL statements at compilation time to avoid
+ // expensive runtime operations. If set to an unsupported dialect, no validation will be done on the statement.
+ // We support the following dialect: ansi, hive.
+ enum Dialect {
+ UNDEFINED = 0;
+ ANSI = 1;
+ HIVE = 2;
+ OTHER = 3;
+ }
+ Dialect dialect = 2;
+}
diff --git a/flyteidl2/core/types.proto b/flyteidl2/core/types.proto
new file mode 100644
index 00000000000..c21bbc29200
--- /dev/null
+++ b/flyteidl2/core/types.proto
@@ -0,0 +1,208 @@
+syntax = "proto3";
+
+package flyteidl2.core;
+
+import "google/protobuf/struct.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/core";
+
+// Define a set of simple types.
+enum SimpleType {
+ NONE = 0;
+ INTEGER = 1;
+ FLOAT = 2;
+ STRING = 3;
+ BOOLEAN = 4;
+ DATETIME = 5;
+ DURATION = 6;
+ BINARY = 7;
+ ERROR = 8;
+ STRUCT = 9;
+}
+
+// Defines schema columns and types to strongly type-validate schemas interoperability.
+message SchemaType {
+ message SchemaColumn {
+ // A unique name -within the schema type- for the column
+ string name = 1;
+
+ enum SchemaColumnType {
+ INTEGER = 0;
+ FLOAT = 1;
+ STRING = 2;
+ BOOLEAN = 3;
+ DATETIME = 4;
+ DURATION = 5;
+ }
+
+ // The column type. This allows a limited set of types currently.
+ SchemaColumnType type = 2;
+ }
+
+ // A list of ordered columns this schema comprises of.
+ repeated SchemaColumn columns = 3;
+}
+
+message StructuredDatasetType {
+ message DatasetColumn {
+ // A unique name within the schema type for the column.
+ string name = 1;
+
+ // The column type.
+ LiteralType literal_type = 2;
+ }
+
+ // A list of ordered columns this schema comprises of.
+ repeated DatasetColumn columns = 1;
+
+ // This is the storage format, the format of the bits at rest
+ // parquet, feather, csv, etc.
+ // For two types to be compatible, the format will need to be an exact match.
+ string format = 2;
+
+ // This is a string representing the type that the bytes in external_schema_bytes are formatted in.
+ // This is an optional field that will not be used for type checking.
+ string external_schema_type = 3;
+
+ // The serialized bytes of a third-party schema library like Arrow.
+ // This is an optional field that will not be used for type checking.
+ bytes external_schema_bytes = 4;
+}
+
+// Defines type behavior for blob objects
+message BlobType {
+ enum BlobDimensionality {
+ SINGLE = 0;
+ MULTIPART = 1;
+ }
+
+ // Format can be a free form string understood by SDK/UI etc like
+ // csv, parquet etc
+ string format = 1;
+ BlobDimensionality dimensionality = 2;
+}
+
+// Enables declaring enum types, with predefined string values
+// For len(values) > 0, the first value in the ordered list is regarded as the default value. If you wish
+// To provide no defaults, make the first value as undefined.
+message EnumType {
+ // Predefined set of enum values.
+ repeated string values = 1;
+}
+
+// Defines a tagged union type, also known as a variant (and formally as the sum type).
+//
+// A sum type S is defined by a sequence of types (A, B, C, ...), each tagged by a string tag
+// A value of type S is constructed from a value of any of the variant types. The specific choice of type is recorded by
+// storing the varaint's tag with the literal value and can be examined in runtime.
+//
+// Type S is typically written as
+// S := Apple A | Banana B | Cantaloupe C | ...
+//
+// Notably, a nullable (optional) type is a sum type between some type X and the singleton type representing a null-value:
+// Optional X := X | Null
+//
+// See also: https://en.wikipedia.org/wiki/Tagged_union
+message UnionType {
+ // Predefined set of variants in union.
+ repeated LiteralType variants = 1;
+}
+
+// Hints to improve type matching
+// e.g. allows distinguishing output from custom type transformers
+// even if the underlying IDL serialization matches.
+message TypeStructure {
+ // Must exactly match for types to be castable
+ string tag = 1;
+ // dataclass_type only exists for dataclasses.
+ // This is used to resolve the type of the fields of dataclass
+ // The key is the field name, and the value is the literal type of the field
+ // e.g. For dataclass Foo, with fields a, and a is a string
+ // Foo.a will be resolved as a literal type of string from dataclass_type
+ map dataclass_type = 2;
+}
+
+// TypeAnnotation encapsulates registration time information about a type. This can be used for various control-plane operations. TypeAnnotation will not be available at runtime when a task runs.
+message TypeAnnotation {
+ // A arbitrary JSON payload to describe a type.
+ google.protobuf.Struct annotations = 1;
+}
+
+// Defines a strong type to allow type checking between interfaces.
+message LiteralType {
+ oneof type {
+ // A simple type that can be compared one-to-one with another.
+ SimpleType simple = 1;
+
+ // A complex type that requires matching of inner fields.
+ SchemaType schema = 2;
+
+ // Defines the type of the value of a collection. Only homogeneous collections are allowed.
+ LiteralType collection_type = 3;
+
+ // Defines the type of the value of a map type. The type of the key is always a string.
+ LiteralType map_value_type = 4;
+
+ // A blob might have specialized implementation details depending on associated metadata.
+ BlobType blob = 5;
+
+ // Defines an enum with pre-defined string values.
+ EnumType enum_type = 7;
+
+ // Generalized schema support
+ StructuredDatasetType structured_dataset_type = 8;
+
+ // Defines an union type with pre-defined LiteralTypes.
+ UnionType union_type = 10;
+ }
+
+ // This field contains type metadata that is descriptive of the type, but is NOT considered in type-checking. This might be used by
+ // consumers to identify special behavior or display extended information for the type.
+ google.protobuf.Struct metadata = 6;
+
+ // This field contains arbitrary data that might have special semantic
+ // meaning for the client but does not effect internal flyte behavior.
+ TypeAnnotation annotation = 9;
+
+ // Hints to improve type matching.
+ TypeStructure structure = 11;
+}
+
+// A reference to an output produced by a node. The type can be retrieved -and validated- from
+// the underlying interface of the node.
+message OutputReference {
+ // Node id must exist at the graph layer.
+ string node_id = 1;
+
+ // Variable name must refer to an output variable for the node.
+ string var = 2;
+
+ repeated PromiseAttribute attr_path = 3;
+}
+
+// PromiseAttribute stores the attribute path of a promise, which will be resolved at runtime.
+// The attribute path is a list of strings and integers.
+// In the following example,
+// ```
+// @workflow
+// def wf():
+// o = t1()
+// t2(o.a["b"][0])
+// ```
+// the output reference t2 binds to has a list of PromiseAttribute ["a", "b", 0]
+
+message PromiseAttribute {
+ oneof value {
+ string string_value = 1;
+ int32 int_value = 2;
+ }
+}
+
+// Represents an error thrown from a node.
+message Error {
+ // The node id that threw the error.
+ string failed_node_id = 1;
+
+ // Error message thrown.
+ string message = 2;
+}
diff --git a/flyteidl2/dataproxy/dataproxy_service.proto b/flyteidl2/dataproxy/dataproxy_service.proto
new file mode 100644
index 00000000000..cac61070a00
--- /dev/null
+++ b/flyteidl2/dataproxy/dataproxy_service.proto
@@ -0,0 +1,220 @@
+syntax = "proto3";
+
+package flyteidl2.dataproxy;
+
+import "buf/validate/validate.proto";
+import "flyteidl2/app/app_definition.proto";
+import "flyteidl2/common/identifier.proto";
+import "flyteidl2/common/run.proto";
+import "flyteidl2/logs/dataplane/payload.proto";
+import "flyteidl2/task/common.proto";
+import "flyteidl2/task/task_definition.proto";
+import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/dataproxy";
+
+// ArtifactType defines the type of artifact to be downloaded.
+enum ArtifactType {
+ ARTIFACT_TYPE_UNSPECIFIED = 0;
+ // ARTIFACT_TYPE_REPORT refers to the HTML report file optionally generated after a task finishes executing.
+ ARTIFACT_TYPE_REPORT = 1;
+ // ARTIFACT_TYPE_CODE_BUNDLE refers to the code bundle (tarball) generated for an action.
+ ARTIFACT_TYPE_CODE_BUNDLE = 2;
+}
+
+// DataProxyService provides an interface for managing data uploads and downloads.
+service DataProxyService {
+ // CreateUploadLocation generates a signed URL for uploading data to the configured storage backend.
+ rpc CreateUploadLocation(CreateUploadLocationRequest) returns (CreateUploadLocationResponse) {}
+
+ rpc UploadInputs(UploadInputsRequest) returns (UploadInputsResponse) {}
+
+ // CreateDownloadLink generates signed URL(s) for downloading a given artifact.
+ rpc CreateDownloadLink(CreateDownloadLinkRequest) returns (CreateDownloadLinkResponse) {}
+
+ // Get input and output data for an action.
+ rpc GetActionData(GetActionDataRequest) returns (GetActionDataResponse) {
+ option idempotency_level = NO_SIDE_EFFECTS;
+ }
+
+ // Stream logs for an action attempt.
+ rpc TailLogs(TailLogsRequest) returns (stream TailLogsResponse) {}
+}
+
+// CreateUploadLocationRequest specifies the request for the CreateUploadLocation API.
+// The data proxy service will generate a storage location with server-side configured prefixes.
+// The generated path follows one of these patterns:
+// - project/domain/(deterministic hash of content_md5)/filename (if filename is present); OR
+// - project/domain/filename_root/filename (if filename_root and filename are present).
+message CreateUploadLocationRequest {
+ // Project to create the upload location for.
+ // +required
+ string project = 1 [(buf.validate.field).string.min_len = 1];
+
+ // Domain to create the upload location for.
+ // +required
+ string domain = 2 [(buf.validate.field).string.min_len = 1];
+
+ // Filename specifies the desired suffix for the generated location. E.g. `file.py` or `pre/fix/file.zip`.
+ // +optional. By default, the service generates a consistent name based on the default filename length provided in the global config..
+ string filename = 3;
+
+ // ExpiresIn defines the requested expiration duration for the generated URL. The request will be rejected if this
+ // exceeds the platform's configured maximum.
+ // +optional. The default value comes from the global config.
+ google.protobuf.Duration expires_in = 4;
+
+ // ContentMD5 restricts the upload location to the specific MD5 provided. The MD5 hash will also appear in the
+ // generated path for verification.
+ // +required
+ bytes content_md5 = 5 [(buf.validate.field).bytes.len = 16];
+
+ // FilenameRoot, if present, will be used instead of the MD5 hash in the path. When combined with filename,
+ // this makes the upload location deterministic. The native URL will still be prefixed by the upload location prefix
+ // configured in the data proxy. This option is useful when uploading multiple related files.
+ // +optional
+ string filename_root = 6;
+
+ // If true, the data proxy will add content_md5 to the Signed URL requirements,
+ // forcing clients to send this checksum with the object.
+ // This is required to enforce data integrity on backends like GCP, ensuring that
+ // the uploaded file matches the hash.
+ bool add_content_md5_metadata = 7;
+
+ // Org is the organization key applied to the resource.
+ // +optional
+ string org = 8;
+
+ // ContentLength specifies the size of the content to be uploaded in bytes.
+ // This is validated against the platform's maximum upload size if provided.
+ // +optional
+ int64 content_length = 9;
+}
+
+// CreateUploadLocationResponse specifies the response for the CreateUploadLocation API.
+message CreateUploadLocationResponse {
+ // SignedUrl is the URL to use for uploading content (e.g. https://my-bucket.s3.amazonaws.com/randomstring/suffix.tar?X-...).
+ string signed_url = 1;
+
+ // NativeUrl is the URL in the format of the configured storage provider (e.g. s3://my-bucket/randomstring/suffix.tar).
+ string native_url = 2;
+
+ // ExpiresAt defines when the signed URL will expire.
+ google.protobuf.Timestamp expires_at = 3;
+
+ // Headers are generated by the data proxy for the client. Clients must include these headers in the upload request.
+ map headers = 4;
+}
+
+message UploadInputsRequest {
+ oneof id {
+ option (buf.validate.oneof).required = true;
+
+ // The user provided run id.
+ common.RunIdentifier run_id = 1;
+
+ // The project id for this run. Run name will be generated.
+ common.ProjectIdentifier project_id = 2;
+ }
+
+ // The task these inputs are used for.
+ // This is used to determine which inputs can be excluded from the hash used for cache key computation in a subsequent CreateRun call.
+ oneof task {
+ option (buf.validate.oneof).required = true;
+
+ // The task id to use.
+ task.TaskIdentifier task_id = 3;
+
+ // The task spec to use.
+ task.TaskSpec task_spec = 4;
+
+ // The trigger name to use.
+ common.TriggerName trigger_name = 5;
+ }
+
+ // The actual inputs.
+ task.Inputs inputs = 6;
+}
+
+message UploadInputsResponse {
+ common.OffloadedInputData offloaded_input_data = 1;
+}
+
+// PreSignedURLs contains a list of signed URLs for downloading artifacts.
+message PreSignedURLs {
+ // SignedUrl are the pre-signed URLs for downloading the artifact.
+ repeated string signed_url = 1;
+
+ // ExpiresAt defines when the signed URLs will expire.
+ google.protobuf.Timestamp expires_at = 2;
+}
+
+// CreateDownloadLinkRequest specifies the request for the CreateDownloadLink API.
+message CreateDownloadLinkRequest {
+ // ArtifactType is the type of artifact to download.
+ // +required
+ ArtifactType artifact_type = 1 [(buf.validate.field).enum = {
+ not_in: [0]
+ }];
+
+ // ExpiresIn defines the requested expiration duration for the generated URLs. The request will be
+ // rejected if this exceeds the platform's configured maximum.
+ // +optional. The default value comes from the global config.
+ google.protobuf.Duration expires_in = 2;
+
+ // Source identifies the action attempt whose artifact is to be downloaded.
+ oneof source {
+ option (buf.validate.oneof).required = true;
+
+ // ActionAttemptId identifies the specific attempt of a run action that produced the artifact.
+ common.ActionAttemptIdentifier action_attempt_id = 3;
+ flyteidl2.app.Identifier app_id = 4;
+ flyteidl2.task.TaskIdentifier task_id = 5;
+ }
+}
+
+// CreateDownloadLinkResponse specifies the response for the CreateDownloadLink API.
+message CreateDownloadLinkResponse {
+ // PreSignedUrls contains the signed URLs and their expiration time.
+ PreSignedURLs pre_signed_urls = 1;
+}
+
+// Request message for querying action data.
+message GetActionDataRequest {
+ // Action to query.
+ common.ActionIdentifier action_id = 1 [(buf.validate.field).required = true];
+}
+
+// Response message for querying action data.
+message GetActionDataResponse {
+ // Inputs for the action.
+ task.Inputs inputs = 1;
+
+ // Outputs for the action.
+ task.Outputs outputs = 2;
+}
+
+// Request message for tailing logs.
+message TailLogsRequest {
+ // The action id.
+ common.ActionIdentifier action_id = 1 [(buf.validate.field).required = true];
+
+ // The attempt number.
+ uint32 attempt = 2 [(buf.validate.field).uint32.gt = 0];
+
+ // The pod name to tail logs from. If not provided, attempt to find the primary pod, else assume the first pod.
+ string pod_name = 3;
+}
+
+// Reponse message for tailing logs.
+message TailLogsResponse {
+ // A batch of logs.
+ message Logs {
+ // Structured log lines.
+ repeated flyteidl2.logs.dataplane.LogLine lines = 1;
+ }
+
+ // One or more batches of logs.
+ repeated Logs logs = 1;
+}
diff --git a/flyteidl2/event/cloudevents.proto b/flyteidl2/event/cloudevents.proto
new file mode 100644
index 00000000000..3d1144384a4
--- /dev/null
+++ b/flyteidl2/event/cloudevents.proto
@@ -0,0 +1,79 @@
+syntax = "proto3";
+
+package flyteidl2.event;
+
+import "flyteidl2/core/artifact_id.proto";
+import "flyteidl2/core/identifier.proto";
+import "flyteidl2/core/interface.proto";
+import "flyteidl2/event/event.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/event";
+
+// This is the cloud event parallel to the raw WorkflowExecutionEvent message. It's filled in with additional
+// information that downstream consumers may find useful.
+message CloudEventWorkflowExecution {
+ event.WorkflowExecutionEvent raw_event = 1;
+
+ core.TypedInterface output_interface = 2;
+
+ // The following are ExecutionMetadata fields
+ // We can't have the ExecutionMetadata object directly because of import cycle
+ repeated core.ArtifactID artifact_ids = 3;
+ core.WorkflowExecutionIdentifier reference_execution = 4;
+ string principal = 5;
+
+ // The ID of the LP that generated the execution that generated the Artifact.
+ // Here for provenance information.
+ // Launch plan IDs are easier to get than workflow IDs so we'll use these for now.
+ core.Identifier launch_plan_id = 6;
+
+ // We can't have the ExecutionMetadata object directly because of import cycle
+ map labels = 7;
+}
+
+message CloudEventNodeExecution {
+ event.NodeExecutionEvent raw_event = 1;
+
+ // The relevant task execution if applicable
+ core.TaskExecutionIdentifier task_exec_id = 2;
+
+ // The typed interface for the task that produced the event.
+ core.TypedInterface output_interface = 3;
+
+ // The following are ExecutionMetadata fields
+ // We can't have the ExecutionMetadata object directly because of import cycle
+ repeated core.ArtifactID artifact_ids = 4;
+ string principal = 5;
+
+ // The ID of the LP that generated the execution that generated the Artifact.
+ // Here for provenance information.
+ // Launch plan IDs are easier to get than workflow IDs so we'll use these for now.
+ core.Identifier launch_plan_id = 6;
+
+ // We can't have the ExecutionMetadata object directly because of import cycle
+ map labels = 7;
+}
+
+message CloudEventTaskExecution {
+ event.TaskExecutionEvent raw_event = 1;
+ // We can't have the ExecutionMetadata object directly because of import cycle
+ map labels = 2;
+}
+
+// This event is to be sent by Admin after it creates an execution.
+message CloudEventExecutionStart {
+ // The execution created.
+ core.WorkflowExecutionIdentifier execution_id = 1;
+ // The launch plan used.
+ core.Identifier launch_plan_id = 2;
+
+ core.Identifier workflow_id = 3;
+
+ // Artifact inputs to the workflow execution for which we have the full Artifact ID. These are likely the result of artifact queries that are run.
+ repeated core.ArtifactID artifact_ids = 4;
+
+ // Artifact inputs to the workflow execution for which we only have the tracking bit that's installed into the Literal's metadata by the Artifact service.
+ repeated string artifact_trackers = 5;
+
+ string principal = 6;
+}
diff --git a/flyteidl2/event/event.proto b/flyteidl2/event/event.proto
new file mode 100644
index 00000000000..526efde0524
--- /dev/null
+++ b/flyteidl2/event/event.proto
@@ -0,0 +1,326 @@
+syntax = "proto3";
+
+package flyteidl2.event;
+
+import "flyteidl2/core/catalog.proto";
+import "flyteidl2/core/execution.proto";
+import "flyteidl2/core/identifier.proto";
+import "flyteidl2/core/literals.proto";
+import "google/protobuf/struct.proto";
+import "google/protobuf/timestamp.proto";
+
+option go_package = "github.com/flyteorg/flyte/v2/gen/go/flyteidl2/event";
+
+message WorkflowExecutionEvent {
+ // Workflow execution id
+ core.WorkflowExecutionIdentifier execution_id = 1;
+
+ // the id of the originator (Propeller) of the event
+ string producer_id = 2;
+
+ core.WorkflowExecution.Phase phase = 3;
+
+ // This timestamp represents when the original event occurred, it is generated
+ // by the executor of the workflow.
+ google.protobuf.Timestamp occurred_at = 4;
+
+ oneof output_result {
+ // URL to the output of the execution, it encodes all the information
+ // including Cloud source provider. ie., s3://...
+ string output_uri = 5;
+
+ // Error information for the execution
+ core.ExecutionError error = 6;
+
+ // Raw output data produced by this workflow execution.
+ core.LiteralMap output_data = 7;
+ }
+}
+
+message NodeExecutionEvent {
+ // Unique identifier for this node execution
+ core.NodeExecutionIdentifier id = 1;
+
+ // the id of the originator (Propeller) of the event
+ string producer_id = 2;
+
+ core.NodeExecution.Phase phase = 3;
+
+ // This timestamp represents when the original event occurred, it is generated
+ // by the executor of the node.
+ google.protobuf.Timestamp occurred_at = 4;
+
+ oneof input_value {
+ string input_uri = 5;
+
+ // Raw input data consumed by this node execution.
+ core.LiteralMap input_data = 20;
+ }
+
+ oneof output_result {
+ // URL to the output of the execution, it encodes all the information
+ // including Cloud source provider. ie., s3://...
+ string output_uri = 6;
+
+ // Error information for the execution
+ core.ExecutionError error = 7;
+
+ // Raw output data produced by this node execution.
+ core.LiteralMap output_data = 15;
+ }
+
+ // Additional metadata to do with this event's node target based
+ // on the node type
+ oneof target_metadata {
+ WorkflowNodeMetadata workflow_node_metadata = 8;
+ TaskNodeMetadata task_node_metadata = 14;
+ }
+
+ // [To be deprecated] Specifies which task (if any) launched this node.
+ ParentTaskExecutionMetadata parent_task_metadata = 9;
+
+ // Specifies the parent node of the current node execution. Node executions at level zero will not have a parent node.
+ ParentNodeExecutionMetadata parent_node_metadata = 10;
+
+ // Retry group to indicate grouping of nodes by retries
+ string retry_group = 11;
+
+ // Identifier of the node in the original workflow/graph
+ // This maps to value of WorkflowTemplate.nodes[X].id
+ string spec_node_id = 12;
+
+ // Friendly readable name for the node
+ string node_name = 13;
+
+ int32 event_version = 16;
+
+ // Whether this node launched a subworkflow.
+ bool is_parent = 17;
+
+ // Whether this node yielded a dynamic workflow.
+ bool is_dynamic = 18;
+
+ // String location uniquely identifying where the deck HTML file is
+ // NativeUrl specifies the url in the format of the configured storage provider (e.g. s3://my-bucket/randomstring/suffix.tar)
+ string deck_uri = 19;
+
+ // This timestamp represents the instant when the event was reported by the executing framework. For example,
+ // when first processing a node the `occurred_at` timestamp should be the instant propeller makes progress, so when
+ // literal inputs are initially copied. The event however will not be sent until after the copy completes.
+ // Extracting both of these timestamps facilitates a more accurate portrayal of the evaluation time-series.
+ google.protobuf.Timestamp reported_at = 21;
+
+ // Indicates if this node is an ArrayNode.
+ bool is_array = 22;
+
+ // So that Admin doesn't have to rebuild the node execution graph to find the target entity, propeller will fill this
+ // in optionally - currently this is only filled in for subworkflows. This is the ID of the subworkflow corresponding
+ // to this node execution. It is difficult to find because Admin only sees one node at a time. A subworkflow could be
+ // nested multiple layers deep, and you'd need to access the correct workflow template to know the target subworkflow.
+ core.Identifier target_entity = 23;
+
+ // Tasks and subworkflows (but not launch plans) that are run within a dynamic task are effectively independent of
+ // the tasks that are registered in Admin's db. Confusingly, they are often identical, but sometimes they are not
+ // even registered at all. Similar to the target_entity field, at the time Admin receives this event, it has no idea
+ // if the relevant execution entity is was registered, or dynamic. This field indicates that the target_entity ID,
+ // as well as task IDs in any corresponding Task Executions, should not be used to looked up the task in Admin's db.
+ bool is_in_dynamic_chain = 24;
+
+ // Whether this node launched an eager task.
+ bool is_eager = 25;
+}
+
+// For Workflow Nodes we need to send information about the workflow that's launched
+message WorkflowNodeMetadata {
+ core.WorkflowExecutionIdentifier execution_id = 1;
+}
+
+message TaskNodeMetadata {
+ // Captures the status of caching for this execution.
+ core.CatalogCacheStatus cache_status = 1;
+ // This structure carries the catalog artifact information
+ core.CatalogMetadata catalog_key = 2;
+ // Captures the status of cache reservations for this execution.
+ core.CatalogReservation.Status reservation_status = 3;
+ // The latest checkpoint location
+ string checkpoint_uri = 4;
+}
+
+message ParentTaskExecutionMetadata {
+ core.TaskExecutionIdentifier id = 1;
+}
+
+message ParentNodeExecutionMetadata {
+ // Unique identifier of the parent node id within the execution
+ // This is value of core.NodeExecutionIdentifier.node_id of the parent node
+ string node_id = 1;
+}
+
+message EventReason {
+ // An explanation for this event
+ string reason = 1;
+
+ // The time this reason occurred
+ google.protobuf.Timestamp occurred_at = 2;
+}
+
+// Plugin specific execution event information. For tasks like Python, Hive, Spark, DynamicJob.
+message TaskExecutionEvent {
+ // ID of the task. In combination with the retryAttempt this will indicate
+ // the task execution uniquely for a given parent node execution.
+ core.Identifier task_id = 1;
+
+ // A task execution is always kicked off by a node execution, the event consumer
+ // will use the parent_id to relate the task to it's parent node execution
+ core.NodeExecutionIdentifier parent_node_execution_id = 2;
+
+ // retry attempt number for this task, ie., 2 for the second attempt
+ uint32 retry_attempt = 3;
+
+ // Phase associated with the event
+ core.TaskExecution.Phase phase = 4;
+
+ // id of the process that sent this event, mainly for trace debugging
+ string producer_id = 5;
+
+ // log information for the task execution
+ repeated core.TaskLog logs = 6;
+
+ // This timestamp represents when the original event occurred, it is generated
+ // by the executor of the task.
+ google.protobuf.Timestamp occurred_at = 7;
+
+ oneof input_value {
+ // URI of the input file, it encodes all the information
+ // including Cloud source provider. ie., s3://...
+ string input_uri = 8;
+
+ // Raw input data consumed by this task execution.
+ core.LiteralMap input_data = 19;
+ }
+
+ oneof output_result {
+ // URI to the output of the execution, it will be in a format that encodes all the information
+ // including Cloud source provider. ie., s3://...
+ string output_uri = 9;
+
+ // Error information for the execution
+ core.ExecutionError error = 10;
+
+ // Raw output data produced by this task execution.
+ core.LiteralMap output_data = 17;
+ }
+
+ // Custom data that the task plugin sends back. This is extensible to allow various plugins in the system.
+ google.protobuf.Struct custom_info = 11;
+
+ // Some phases, like RUNNING, can send multiple events with changed metadata (new logs, additional custom_info, etc)
+ // that should be recorded regardless of the lack of phase change.
+ // The version field should be incremented when metadata changes across the duration of an individual phase.
+ uint32 phase_version = 12;
+
+ // An optional explanation for the phase transition.
+ // Deprecated: Use reasons instead.
+ string reason = 13 [deprecated = true];
+
+ // An optional list of explanations for the phase transition.
+ repeated EventReason reasons = 21;
+
+ // A predefined yet extensible Task type identifier. If the task definition is already registered in flyte admin
+ // this type will be identical, but not all task executions necessarily use pre-registered definitions and this
+ // type is useful to render the task in the UI, filter task executions, etc.
+ string task_type = 14;
+
+ // Metadata around how a task was executed.
+ TaskExecutionMetadata metadata = 16;
+
+ // The event version is used to indicate versioned changes in how data is reported using this
+ // proto message. For example, event_verison > 0 means that maps tasks report logs using the
+ // TaskExecutionMetadata ExternalResourceInfo fields for each subtask rather than the TaskLog
+ // in this message.
+ int32 event_version = 18;
+
+ // This timestamp represents the instant when the event was reported by the executing framework. For example, a k8s
+ // pod task may be marked completed at (ie. `occurred_at`) the instant the container running user code completes,
+ // but this event will not be reported until the pod is marked as completed. Extracting both of these timestamps
+ // facilitates a more accurate portrayal of the evaluation time-series.
+ google.protobuf.Timestamp reported_at = 20;
+
+ // Contains metadata required to identify logs related to this task execution
+ core.LogContext log_context = 22;
+}
+
+// This message contains metadata about external resources produced or used by a specific task execution.
+message ExternalResourceInfo {
+ // Identifier for an external resource created by this task execution, for example Qubole query ID or presto query ids.
+ string external_id = 1;
+
+ // A unique index for the external resource with respect to all external resources for this task. Although the
+ // identifier may change between task reporting events or retries, this will remain the same to enable aggregating
+ // information from multiple reports.
+ uint32 index = 2;
+
+ // Retry attempt number for this external resource, ie., 2 for the second attempt
+ uint32 retry_attempt = 3;
+
+ // Phase associated with the external resource
+ core.TaskExecution.Phase phase = 4;
+
+ // Captures the status of caching for this external resource execution.
+ core.CatalogCacheStatus cache_status = 5;
+
+ // log information for the external resource execution
+ repeated core.TaskLog logs = 6;
+
+ // Additional metadata to do with this event's node target based on the node type. We are
+ // explicitly not including the task_node_metadata here because it is not clear if it is needed.
+ // If we decide to include in the future, we should deprecate the cache_status field.
+ oneof target_metadata {
+ WorkflowNodeMetadata workflow_node_metadata = 7;
+ }
+
+ // Extensible field for custom, plugin-specific info
+ google.protobuf.Struct custom_info = 8;
+
+ // Contains metadata required to identify logs related to this task execution
+ core.LogContext log_context = 9;
+}
+
+// This message holds task execution metadata specific to resource allocation used to manage concurrent
+// executions for a project namespace.
+message ResourcePoolInfo {
+ // Unique resource ID used to identify this execution when allocating a token.
+ string allocation_token = 1;
+
+ // Namespace under which this task execution requested an allocation token.
+ string namespace = 2;
+}
+
+// Holds metadata around how a task was executed.
+// As a task transitions across event phases during execution some attributes, such its generated name, generated external resources,
+// and more may grow in size but not change necessarily based on the phase transition that sparked the event update.
+// Metadata is a container for these attributes across the task execution lifecycle.
+message TaskExecutionMetadata {
+ // Unique, generated name for this task execution used by the backend.
+ string generated_name = 1;
+
+ // Additional data on external resources on other back-ends or platforms (e.g. Hive, Qubole, etc) launched by this task execution.
+ repeated ExternalResourceInfo external_resources = 2;
+
+ // Includes additional data on concurrent resource management used during execution..
+ // This is a repeated field because a plugin can request multiple resource allocations during execution.
+ repeated ResourcePoolInfo resource_pool_info = 3;
+
+ // The identifier of the plugin used to execute this task.
+ string plugin_identifier = 4;
+
+ // Includes the broad category of machine used for this specific task execution.
+ enum InstanceClass {
+ // The default instance class configured for the flyte application platform.
+ DEFAULT = 0;
+
+ // The instance class configured for interruptible tasks.
+ INTERRUPTIBLE = 1;
+ }
+ InstanceClass instance_class = 16;
+}
diff --git a/flyteidl2/gen_utils/python/README.md b/flyteidl2/gen_utils/python/README.md
new file mode 100644
index 00000000000..cae843b7a37
--- /dev/null
+++ b/flyteidl2/gen_utils/python/README.md
@@ -0,0 +1 @@
+# Python library of flyte IDL (protobuf)
\ No newline at end of file
diff --git a/flyteidl2/gen_utils/python/pyproject.toml b/flyteidl2/gen_utils/python/pyproject.toml
new file mode 100644
index 00000000000..5d69d553604
--- /dev/null
+++ b/flyteidl2/gen_utils/python/pyproject.toml
@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools", "setuptools_scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "flyteidl2"
+dynamic = ["version"]
+authors = [{ name = "Union Eng", email = "support@union.ai" }]
+description = "IDL for Flyte"
+readme = { file = "README.md", content-type = "text/markdown" }
+requires-python = ">=3.10"
+dependencies = [
+ 'googleapis-common-protos',
+ 'protoc-gen-openapiv2',
+ 'protobuf>=4.21.1',
+ "protovalidate>=1.0.0",
+]
+classifiers = [
+ "Intended Audience :: Science/Research",
+ "Intended Audience :: Developers",
+ "License :: OSI Approved :: Apache Software License",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
+ "Topic :: Scientific/Engineering",
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
+ "Topic :: Software Development",
+ "Topic :: Software Development :: Libraries",
+ "Topic :: Software Development :: Libraries :: Python Modules",
+]
+license-files = ["licenses/*.txt", "LICENSE"]
+
+[tool.setuptools]
+include-package-data = true
+
+# Intentionally leaving out the google folder, which contains googleapis-common-protos. This library is a dependency
+# of too many google libraries including grpcio-status which flyte already depends on, so don't want to
+# risk version conflicts.
+[tool.setuptools.packages.find]
+include = ["flyteidl2*", "buf*"]
+exclude = ["build*"]
+
+[tool.setuptools_scm]
+root = "../../"
diff --git a/flyteidl2/gen_utils/python/setup.py b/flyteidl2/gen_utils/python/setup.py
new file mode 100644
index 00000000000..7f576889bf8
--- /dev/null
+++ b/flyteidl2/gen_utils/python/setup.py
@@ -0,0 +1,6 @@
+from setuptools import setup
+
+
+__version__ = "0.0.0+develop"
+
+setup()
\ No newline at end of file
diff --git a/flyteidl2/gen_utils/python/uv.lock b/flyteidl2/gen_utils/python/uv.lock
new file mode 100644
index 00000000000..5ae7fb08e49
--- /dev/null
+++ b/flyteidl2/gen_utils/python/uv.lock
@@ -0,0 +1,285 @@
+version = 1
+revision = 3
+requires-python = ">=3.10"
+resolution-markers = [
+ "python_full_version == '3.13.*'",
+ "python_full_version == '3.12.*'",
+ "python_full_version < '3.12' or python_full_version >= '3.14'",
+]
+
+[[package]]
+name = "bufbuild-protovalidate-protocolbuffers-pyi"
+version = "32.1.0.1.20240401165935+b983156c5e99"
+source = { registry = "https://buf.build/gen/python" }
+dependencies = [
+ { name = "protobuf" },
+ { name = "types-protobuf" },
+]
+wheels = [
+ { url = "https://buf.build/gen/python/bufbuild-protovalidate-protocolbuffers-pyi/bufbuild_protovalidate_protocolbuffers_pyi-32.1.0.1.20240401165935+b983156c5e99-py3-none-any.whl" },
+]
+
+[[package]]
+name = "bufbuild-protovalidate-protocolbuffers-python"
+version = "32.0.0.1.20250912141014+52f32327d4b0"
+source = { registry = "https://buf.build/gen/python" }
+dependencies = [
+ { name = "bufbuild-protovalidate-protocolbuffers-pyi" },
+ { name = "protobuf" },
+]
+wheels = [
+ { url = "https://buf.build/gen/python/bufbuild-protovalidate-protocolbuffers-python/bufbuild_protovalidate_protocolbuffers_python-32.0.0.1.20250912141014+52f32327d4b0-py3-none-any.whl" },
+]
+
+[[package]]
+name = "cel-python"
+version = "0.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "jmespath" },
+ { name = "lark" },
+ { name = "python-dateutil" },
+ { name = "pyyaml" },
+ { name = "types-python-dateutil" },
+ { name = "types-pyyaml" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/10/87/85a1b99b98f6466bb87d40df636626385945ae82348e82cd97d44313f612/cel_python-0.2.0.tar.gz", hash = "sha256:75de72a5cf223ec690b236f0cc24da267219e667bd3e7f8f4f20595fcc1c0c0f", size = 67185, upload-time = "2025-02-14T11:42:21.882Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/18/28/08871462a0347b3e707658a8308be6f979167488a2196f93b402c2ea7170/cel_python-0.2.0-py3-none-any.whl", hash = "sha256:478ff73def7b39d51e6982f95d937a57c2b088c491c578fe5cecdbd79f476f60", size = 71337, upload-time = "2025-02-14T11:42:19.996Z" },
+]
+
+[[package]]
+name = "flyteidl2"
+source = { editable = "." }
+dependencies = [
+ { name = "bufbuild-protovalidate-protocolbuffers-pyi" },
+ { name = "bufbuild-protovalidate-protocolbuffers-python" },
+ { name = "googleapis-common-protos" },
+ { name = "protobuf" },
+ { name = "protoc-gen-openapiv2" },
+ { name = "protovalidate" },
+]
+
+[package.metadata]
+requires-dist = [
+ { name = "bufbuild-protovalidate-protocolbuffers-pyi", specifier = "==32.1.0.1.20240401165935+b983156c5e99", index = "https://buf.build/gen/python" },
+ { name = "bufbuild-protovalidate-protocolbuffers-python", specifier = "==32.0.0.1.20250912141014+52f32327d4b0", index = "https://buf.build/gen/python" },
+ { name = "googleapis-common-protos" },
+ { name = "protobuf", specifier = ">=4.21.1" },
+ { name = "protoc-gen-openapiv2" },
+ { name = "protovalidate", specifier = ">=1.0.0" },
+]
+
+[[package]]
+name = "google-re2"
+version = "1.1.20250805"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/93/ed/7caa25b34a201ef8db5a635e03ca71c926caff92aba1b17e86b78190de43/google_re2-1.1.20250805.tar.gz", hash = "sha256:c55d9f7c92a814eb53918a7b38e5ba5eaa1c99548321acb826da9532781af5b5", size = 11698, upload-time = "2025-08-05T19:30:24.345Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/92/9c/e120dc14daa0b6d5e0ddb659e0f132292abf22fad3563c017b73ab549a01/google_re2-1.1.20250805-1-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:7d2a7dea1448733184a99516a41f28ffcfc9eda345697a14fd5c6d8144b60841", size = 483036, upload-time = "2025-08-05T19:29:05.84Z" },
+ { url = "https://files.pythonhosted.org/packages/b2/b6/981ae8734410617c0cb6d32b059a6833ca980bbe6e65840bc8e68f5697ea/google_re2-1.1.20250805-1-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:51b4e3c5e6f3f74193666295385b44e1043e55cf98e4b933fe11a0d7a2457a67", size = 514139, upload-time = "2025-08-05T19:29:07.721Z" },
+ { url = "https://files.pythonhosted.org/packages/73/56/d1c1c729b7e356dd8b8224bf1873d90fa94c350eddce0ef7da775440a552/google_re2-1.1.20250805-1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:0c43d2120ba62e8da4d064dcb5b5c9ac103bfe4cd71a5472055a7f02298b544b", size = 484050, upload-time = "2025-08-05T19:29:09.163Z" },
+ { url = "https://files.pythonhosted.org/packages/1d/78/a07b752d7d879f25b6de12f442ebfd57be9acc79857f16c026ba989afad0/google_re2-1.1.20250805-1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:46eb32ca99136e5ff76b33195b37d41342afcc505f4b60309a4159008f80b064", size = 515591, upload-time = "2025-08-05T19:29:10.742Z" },
+ { url = "https://files.pythonhosted.org/packages/0b/4a/651971a6ece6a85ff6598bcf801cb0c7ee20a251b9631b7b5d4886642818/google_re2-1.1.20250805-1-cp310-cp310-macosx_15_0_arm64.whl", hash = "sha256:eaf8b912020ec9bcc1811f64478e2dfb82907e502b718ad4b263045820519595", size = 484613, upload-time = "2025-08-05T19:29:12.348Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/ad/8768a99b2b79b14806b7da89f783f6828f540469169066cf3a7bb16cbe44/google_re2-1.1.20250805-1-cp310-cp310-macosx_15_0_x86_64.whl", hash = "sha256:0df2591355131457f9a008b3bd8fbdb104f8344b75d150fd90ddc0bc19b80935", size = 511041, upload-time = "2025-08-05T19:29:13.87Z" },
+ { url = "https://files.pythonhosted.org/packages/68/34/997ee9e113398c5ca6ec8bb5c2d210e7931e44b1fbd38024d09dabd2ba9a/google_re2-1.1.20250805-1-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fb56c6e7c8fe2eaf045b987dbf8dfc979f61a21e6446dd8b3c0e4028f9ff8611", size = 572722, upload-time = "2025-08-05T19:29:15.435Z" },
+ { url = "https://files.pythonhosted.org/packages/51/e2/f99887835200a961a957dab4208d00ae344a8906ed9e0bb1ce578aec87a1/google_re2-1.1.20250805-1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7f2124ccc4e06c1841e18360edbc379f050b8dcb54096293e2e6e90b5f913f92", size = 588956, upload-time = "2025-08-05T19:29:16.975Z" },
+ { url = "https://files.pythonhosted.org/packages/4c/fc/bd56b2133ce1d6853e4abafa89685860eda725a2fe62c233a30cfe928abb/google_re2-1.1.20250805-1-cp310-cp310-win32.whl", hash = "sha256:0bcf2acdc32a3890ddfca87fa846806b6db200a059a83ab114e6ab390beaf10e", size = 432851, upload-time = "2025-08-05T19:29:18.568Z" },
+ { url = "https://files.pythonhosted.org/packages/a8/ac/21473b4de4a829bc9822d810fc1a26c7abdbee76c60a915a8ade0dae6372/google_re2-1.1.20250805-1-cp310-cp310-win_amd64.whl", hash = "sha256:8c6da22b158459e5a592fcd66a9e99347f517284d91d61b6ff2befff3eb79339", size = 490154, upload-time = "2025-08-05T19:29:19.762Z" },
+ { url = "https://files.pythonhosted.org/packages/ec/6f/c99a04f13114282362bf73dd1a79e96b92ed80ed47bf1f57744646753e08/google_re2-1.1.20250805-1-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:e7f7e960002c702ae2c0aff3b8db895ded37006ac7aa0c158743fb20dcd485c2", size = 483730, upload-time = "2025-08-05T19:29:21.181Z" },
+ { url = "https://files.pythonhosted.org/packages/23/c8/994a5fef87eb977d06ff8dfe7bb59ba31f535d3f2622334b6b32f1b65528/google_re2-1.1.20250805-1-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:bfacaa9b274871796bfd0750e23f804ff632a5397b830c640449d3d57b3d148f", size = 515513, upload-time = "2025-08-05T19:29:22.63Z" },
+ { url = "https://files.pythonhosted.org/packages/30/25/81672efc1d0cce43813e0b3bffc14c3d32e102cf254c9d280bf4e5194664/google_re2-1.1.20250805-1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:c4f72b275ec6b5ef6e3215dab9392aefc2e8f8590d0c3d8b266e6a913503d2a1", size = 485374, upload-time = "2025-08-05T19:29:23.925Z" },
+ { url = "https://files.pythonhosted.org/packages/da/04/7a3361618217d401203a9eb34712c824e9645f7d2830d1e804eb9859374c/google_re2-1.1.20250805-1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:c6b5550a9767e444b17a9c3c4b020c51629282748d77244d833aacbd765f28fe", size = 517105, upload-time = "2025-08-05T19:29:25.196Z" },
+ { url = "https://files.pythonhosted.org/packages/38/97/29e8097e449b0af993acd7d7d658ecfac3914ac501e6c6d40ffcebc03b8e/google_re2-1.1.20250805-1-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:4c1e5ec68dfe2e0866f14468600e2e2928dcfe684c0f2fbeda8d16f14f909141", size = 485756, upload-time = "2025-08-05T19:29:26.975Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/88/cb3647eb92c991d93fde8b1fefddd52ff3bb45a5d1c20185e2fd24e1e5fa/google_re2-1.1.20250805-1-cp311-cp311-macosx_15_0_x86_64.whl", hash = "sha256:336d6830888ea2abdc1744d201e19cf76c4f001cf821bed3e8844c1899bcbdaf", size = 512227, upload-time = "2025-08-05T19:29:28.207Z" },
+ { url = "https://files.pythonhosted.org/packages/b8/fe/33e357bb8d090a4879e9e257be577740d1b5d762686b4fa448f050b15caf/google_re2-1.1.20250805-1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:52a153ce37ccfd5429257a91d68f4338fe2809711bff64c7ab84c97ddef2de25", size = 573694, upload-time = "2025-08-05T19:29:29.52Z" },
+ { url = "https://files.pythonhosted.org/packages/96/97/e1648859b140b76717319a8d8aceec3365e354ea9ff0690d0fbbcb5774e3/google_re2-1.1.20250805-1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ffc79fe2f4be9c5e3e8bb04c8e84e0c9a27b5f08ad5103779b084005e99d220", size = 590760, upload-time = "2025-08-05T19:29:30.967Z" },
+ { url = "https://files.pythonhosted.org/packages/af/0a/1dae50eece5fe4f53503f47f8535ee7f8667e2f84dbbf87f0fa8942678b6/google_re2-1.1.20250805-1-cp311-cp311-win32.whl", hash = "sha256:2cfccbbd8577250406a3a3ff1b5d3ee21a479e3f1a01f78042d4d15c461eca1e", size = 434080, upload-time = "2025-08-05T19:29:32.578Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/b8/1c4e100d2b1dac7c4ab52d18b8c3060a835f1e2b9dc0802024313fa69277/google_re2-1.1.20250805-1-cp311-cp311-win_amd64.whl", hash = "sha256:4d0669fed9f9f993c5cffae42d7193da752e5b4e42e95b0e9ee0b95de9fcd8ad", size = 490954, upload-time = "2025-08-05T19:29:33.784Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/ac/3ed55b2f70cb1b091a59c5242376524e68de88d0280ea0be7c8f4754a37d/google_re2-1.1.20250805-1-cp311-cp311-win_arm64.whl", hash = "sha256:50495e5f783e0c1577384bac75455a799904158b5077284c70e6a9510995f3be", size = 642137, upload-time = "2025-08-05T19:29:34.973Z" },
+ { url = "https://files.pythonhosted.org/packages/78/b5/5ed6ffcb1f33348e7f212819d1082f125bd224c89aef842b78b63f9a97e0/google_re2-1.1.20250805-1-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:39f81ff026701533593ffb43acd999d11b8ff769d2234e2083c9ae38d02a01a4", size = 485610, upload-time = "2025-08-05T19:29:36.243Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/e3/b4db34e633b0c5c880c2a3371eedcab600ea0c04a51a2509cda09c913e1e/google_re2-1.1.20250805-1-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:7d26aecd8850ec469bf8ae7d3a1ad26b1b1c0477400351ff5df181ef5dff68f0", size = 518722, upload-time = "2025-08-05T19:29:37.51Z" },
+ { url = "https://files.pythonhosted.org/packages/b9/96/ca6f0ff5693558ba104007cf7cc8187d3d47c5d01af06c4204bbe9d8d160/google_re2-1.1.20250805-1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f45314ef7b22c28a1c43469d522398068af4bd1b59839c831033723c782c7402", size = 486962, upload-time = "2025-08-05T19:29:38.775Z" },
+ { url = "https://files.pythonhosted.org/packages/e9/e5/41e316d9a160bca99f7a4bff2dc9f4a9b8a3b3b927a610fdf7717f0af2f7/google_re2-1.1.20250805-1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:13e8b83655fcb97d7190d8c07a2886dd5bf9c55935419c98a4b7f09cc6e2019d", size = 520209, upload-time = "2025-08-05T19:29:39.967Z" },
+ { url = "https://files.pythonhosted.org/packages/dc/c6/30d8c5988c640d6de7a9b0739bc33d9b2a4d00bf12a7e6c0abfebc0c365b/google_re2-1.1.20250805-1-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:9c17c678b3bf2ca874c74b898a21b534d3e60123eda8ef18dde1c1932d142b1f", size = 487152, upload-time = "2025-08-05T19:29:41.596Z" },
+ { url = "https://files.pythonhosted.org/packages/73/1f/d3f04e5c66c2bf235cb449f4e7372596375f6a8537eaeaa35f076927bd0b/google_re2-1.1.20250805-1-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:a848f44752286372cfb0ff225a836ed7b02738b29ca31ed3c58e70dc7d584537", size = 514497, upload-time = "2025-08-05T19:29:43.196Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/e3/14c2999896aa5bc111a18fe41c56283f222179076c52d515ea5c3e0043ad/google_re2-1.1.20250805-1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a90f090081e415ee182a01f4113076ad5707c5501ae985c8edc5bfc439cbdee6", size = 572418, upload-time = "2025-08-05T19:29:44.478Z" },
+ { url = "https://files.pythonhosted.org/packages/5f/ad/7c4e61bccbfaf036011960ebbc8743971a34f1bd1b07d1f379b2feb81296/google_re2-1.1.20250805-1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b5749bcbb363cdfdff5da14aa0c53de28f918662d0b6f546af31f96c8fd46dd", size = 591359, upload-time = "2025-08-05T19:29:46.186Z" },
+ { url = "https://files.pythonhosted.org/packages/9f/fe/f1a0966c76a52467879f951ba1ffbbc5f2a4a0ecc2f2c2e40b131613b9e9/google_re2-1.1.20250805-1-cp312-cp312-win32.whl", hash = "sha256:19689531ce9839813035663df68aa49074c92e426b20095e5e665521c55c1cab", size = 433756, upload-time = "2025-08-05T19:29:47.416Z" },
+ { url = "https://files.pythonhosted.org/packages/7e/27/f20d98d5121479eed67127eafb3ed99531d9bc43fac2e75e04938950b2ca/google_re2-1.1.20250805-1-cp312-cp312-win_amd64.whl", hash = "sha256:b533077b8a1c120c5f446e6734893d2a18d098e3edde149dda6a9ff9a3e2e7d2", size = 491663, upload-time = "2025-08-05T19:29:48.726Z" },
+ { url = "https://files.pythonhosted.org/packages/80/5a/8997a1e00fcd75db5a6c1243ee512f174a2e266f0017d2a209de36007cef/google_re2-1.1.20250805-1-cp312-cp312-win_arm64.whl", hash = "sha256:3a0193237b274faf57492efbeecc9be5818f3852f186ea5c672490b49da4d124", size = 642623, upload-time = "2025-08-05T19:29:50.172Z" },
+ { url = "https://files.pythonhosted.org/packages/79/c0/f4b5c894335bc7d276e5be58d72e24811db8d5900dfdf37069528d65b73c/google_re2-1.1.20250805-1-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:049732fce70dea6246f035027e512f7cbc22fa9c7f2969c503cd0abb0fcfc674", size = 485546, upload-time = "2025-08-05T19:29:51.461Z" },
+ { url = "https://files.pythonhosted.org/packages/dd/a8/afcddbf964add57569cc0081e99fc968e1bc9d65c34612e0a63ec2085606/google_re2-1.1.20250805-1-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:552e37f08514778d98b6d27aea2abd80efca19c4812ca6388175fd5e54d08229", size = 518842, upload-time = "2025-08-05T19:29:52.826Z" },
+ { url = "https://files.pythonhosted.org/packages/02/a4/82063779d7c6c56136b8217c022f6d95c490d73ccc78ca5e57158fd91855/google_re2-1.1.20250805-1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6d452dab6bd9dedecef4cc4a0ba22203f5c4273395fd7896fe9ed0cc85643b11", size = 487043, upload-time = "2025-08-05T19:29:54.526Z" },
+ { url = "https://files.pythonhosted.org/packages/5d/06/aee9f7315c00c77e1f3653d44a356c68c2066bc1de516c1bd473b6d827c5/google_re2-1.1.20250805-1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:358f497682cb5d57f0dd107944f7a2167226d31fb5211cfd187ec16cb5275355", size = 520233, upload-time = "2025-08-05T19:29:55.8Z" },
+ { url = "https://files.pythonhosted.org/packages/08/7f/d28e64ce85604635da2870c99ac8c8c3e078aaba1f132ecea239f65f0030/google_re2-1.1.20250805-1-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:af66c822179f7f412e4c1fcc8b5ca84885e24ba77c2ee8aa7364f19131b77e7d", size = 487233, upload-time = "2025-08-05T19:29:57.531Z" },
+ { url = "https://files.pythonhosted.org/packages/ff/37/bc22d185861461aafa8e8193a1e22f9eac1217e9bec9ac34b681f1c10bd9/google_re2-1.1.20250805-1-cp313-cp313-macosx_15_0_x86_64.whl", hash = "sha256:555b6183fa1a6f54117ec0eda333b98d96620c5d59b563a1dbe2a3eddf64ca24", size = 514473, upload-time = "2025-08-05T19:29:58.826Z" },
+ { url = "https://files.pythonhosted.org/packages/56/05/a6da22582817396e30931e0d292d2ef4819c2eea3fb21b08661f9a6f3106/google_re2-1.1.20250805-1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e7dfeab6a641a8e6c8ac1cb7fa40b4d2cb91571a3c4bcc840a6d1245d37c33bb", size = 572370, upload-time = "2025-08-05T19:30:00.164Z" },
+ { url = "https://files.pythonhosted.org/packages/ec/14/feb54f3f11522f334294a916dffb54ab36dfaba47c7c8d0e091aec1084f1/google_re2-1.1.20250805-1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8b328e6d75e7b1e161dd834c42a7e1762d2e03919453257a64712a5bda798226", size = 591407, upload-time = "2025-08-05T19:30:01.62Z" },
+ { url = "https://files.pythonhosted.org/packages/f3/b2/7c785a0596e8535794b5c21d27c3081928748acd556a4a0ef3b2c8338881/google_re2-1.1.20250805-1-cp313-cp313-win32.whl", hash = "sha256:eb1c398eaec3c8ffe74fe7064008c81f452ca5bdf12c2538acc4c087d043a6e1", size = 433845, upload-time = "2025-08-05T19:30:03.46Z" },
+ { url = "https://files.pythonhosted.org/packages/5b/1d/af1fc1a5247d6549e67716bcb5e790761bb550026e659dfcfd89408389fd/google_re2-1.1.20250805-1-cp313-cp313-win_amd64.whl", hash = "sha256:18f72952c71de0ace48fbe0ca9928bd7b7bbe1062f685c0d1326a0205966f2dc", size = 491718, upload-time = "2025-08-05T19:30:05.14Z" },
+ { url = "https://files.pythonhosted.org/packages/eb/89/39fd1d16a9cbda45cf55b920fac4840fc16ca668b96333f1a5d8931e1ec0/google_re2-1.1.20250805-1-cp313-cp313-win_arm64.whl", hash = "sha256:149ec3ce1a4711daf45aab4fc7d5926f9e3082ee2c6ea138043c27e64ff1d782", size = 642609, upload-time = "2025-08-05T19:30:06.632Z" },
+]
+
+[[package]]
+name = "googleapis-common-protos"
+version = "1.70.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "protobuf" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/39/24/33db22342cf4a2ea27c9955e6713140fedd51e8b141b5ce5260897020f1a/googleapis_common_protos-1.70.0.tar.gz", hash = "sha256:0e1b44e0ea153e6594f9f394fef15193a68aaaea2d843f83e2742717ca753257", size = 145903, upload-time = "2025-04-14T10:17:02.924Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/86/f1/62a193f0227cf15a920390abe675f386dec35f7ae3ffe6da582d3ade42c7/googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8", size = 294530, upload-time = "2025-04-14T10:17:01.271Z" },
+]
+
+[[package]]
+name = "jmespath"
+version = "1.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" },
+]
+
+[[package]]
+name = "lark"
+version = "0.12.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/02/1d/29681d27b84e384ea50b5546e9f0089126afbc91754db4ca89593fcfd0e8/lark-0.12.0.tar.gz", hash = "sha256:7da76fcfddadabbbbfd949bbae221efd33938451d90b1fefbbc423c3cccf48ef", size = 235168, upload-time = "2021-11-12T11:15:32.124Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/cb/39/cef2ccdfd984ae3cf93878d050c1b7c9354dd9493ce83fd9bb33a41f7a33/lark-0.12.0-py2.py3-none-any.whl", hash = "sha256:ed1d891cbcf5151ead1c1d14663bf542443e579e63a76ae175b01b899bd854ca", size = 103540, upload-time = "2021-11-12T11:15:34.408Z" },
+]
+
+[[package]]
+name = "protobuf"
+version = "6.32.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/fa/a4/cc17347aa2897568beece2e674674359f911d6fe21b0b8d6268cd42727ac/protobuf-6.32.1.tar.gz", hash = "sha256:ee2469e4a021474ab9baafea6cd070e5bf27c7d29433504ddea1a4ee5850f68d", size = 440635, upload-time = "2025-09-11T21:38:42.935Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/c0/98/645183ea03ab3995d29086b8bf4f7562ebd3d10c9a4b14ee3f20d47cfe50/protobuf-6.32.1-cp310-abi3-win32.whl", hash = "sha256:a8a32a84bc9f2aad712041b8b366190f71dde248926da517bde9e832e4412085", size = 424411, upload-time = "2025-09-11T21:38:27.427Z" },
+ { url = "https://files.pythonhosted.org/packages/8c/f3/6f58f841f6ebafe076cebeae33fc336e900619d34b1c93e4b5c97a81fdfa/protobuf-6.32.1-cp310-abi3-win_amd64.whl", hash = "sha256:b00a7d8c25fa471f16bc8153d0e53d6c9e827f0953f3c09aaa4331c718cae5e1", size = 435738, upload-time = "2025-09-11T21:38:30.959Z" },
+ { url = "https://files.pythonhosted.org/packages/10/56/a8a3f4e7190837139e68c7002ec749190a163af3e330f65d90309145a210/protobuf-6.32.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:d8c7e6eb619ffdf105ee4ab76af5a68b60a9d0f66da3ea12d1640e6d8dab7281", size = 426454, upload-time = "2025-09-11T21:38:34.076Z" },
+ { url = "https://files.pythonhosted.org/packages/3f/be/8dd0a927c559b37d7a6c8ab79034fd167dcc1f851595f2e641ad62be8643/protobuf-6.32.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:2f5b80a49e1eb7b86d85fcd23fe92df154b9730a725c3b38c4e43b9d77018bf4", size = 322874, upload-time = "2025-09-11T21:38:35.509Z" },
+ { url = "https://files.pythonhosted.org/packages/5c/f6/88d77011b605ef979aace37b7703e4eefad066f7e84d935e5a696515c2dd/protobuf-6.32.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:b1864818300c297265c83a4982fd3169f97122c299f56a56e2445c3698d34710", size = 322013, upload-time = "2025-09-11T21:38:37.017Z" },
+ { url = "https://files.pythonhosted.org/packages/97/b7/15cc7d93443d6c6a84626ae3258a91f4c6ac8c0edd5df35ea7658f71b79c/protobuf-6.32.1-py3-none-any.whl", hash = "sha256:2601b779fc7d32a866c6b4404f9d42a3f67c5b9f3f15b4db3cccabe06b95c346", size = 169289, upload-time = "2025-09-11T21:38:41.234Z" },
+]
+
+[[package]]
+name = "protoc-gen-openapiv2"
+version = "0.0.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "googleapis-common-protos" },
+ { name = "protobuf" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d8/d2/84fecd8df61640226c726c12ad7ddd2a7666a7cd7f898b9a5b72e3a66d44/protoc-gen-openapiv2-0.0.1.tar.gz", hash = "sha256:6f79188d842c13177c9c0558845442c340b43011bf67dfef1dfc3bc067506409", size = 7323, upload-time = "2022-12-02T01:40:57.306Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/2d/ac/bd8961859d8f3f81530465d2ce9b165627e961c00348939009bac2700cc6/protoc_gen_openapiv2-0.0.1-py3-none-any.whl", hash = "sha256:18090c8be3877c438e7da0f7eb7cace45a9a210306bca4707708dbad367857be", size = 7883, upload-time = "2022-12-02T01:40:55.244Z" },
+]
+
+[[package]]
+name = "protovalidate"
+version = "1.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "cel-python" },
+ { name = "google-re2" },
+ { name = "protobuf" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/dd/98/1595ae90c4a29c625580ee84415bb0c752e09c6c7aa13595e8ea94a7c929/protovalidate-1.0.0.tar.gz", hash = "sha256:926f7a212fed9190d00cc076fa24ef5e48a404b5577465028697f4dea8c4a507", size = 215286, upload-time = "2025-09-12T16:28:02.665Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/17/1d/30a86726b317593469eb526c8ca25dd8ce7f7b9f4237137fedb1f352ffff/protovalidate-1.0.0-py3-none-any.whl", hash = "sha256:933818942700c85d4a47f1030e61f59d7bd9a8c1572e9dc822f98eef45a39d9e", size = 29478, upload-time = "2025-09-12T16:28:01.201Z" },
+]
+
+[[package]]
+name = "python-dateutil"
+version = "2.9.0.post0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "six" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
+]
+
+[[package]]
+name = "pyyaml"
+version = "6.0.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199, upload-time = "2024-08-06T20:31:40.178Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758, upload-time = "2024-08-06T20:31:42.173Z" },
+ { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463, upload-time = "2024-08-06T20:31:44.263Z" },
+ { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280, upload-time = "2024-08-06T20:31:50.199Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239, upload-time = "2024-08-06T20:31:52.292Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802, upload-time = "2024-08-06T20:31:53.836Z" },
+ { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527, upload-time = "2024-08-06T20:31:55.565Z" },
+ { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052, upload-time = "2024-08-06T20:31:56.914Z" },
+ { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774, upload-time = "2024-08-06T20:31:58.304Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612, upload-time = "2024-08-06T20:32:03.408Z" },
+ { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040, upload-time = "2024-08-06T20:32:04.926Z" },
+ { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829, upload-time = "2024-08-06T20:32:06.459Z" },
+ { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167, upload-time = "2024-08-06T20:32:08.338Z" },
+ { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952, upload-time = "2024-08-06T20:32:14.124Z" },
+ { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301, upload-time = "2024-08-06T20:32:16.17Z" },
+ { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638, upload-time = "2024-08-06T20:32:18.555Z" },
+ { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850, upload-time = "2024-08-06T20:32:19.889Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980, upload-time = "2024-08-06T20:32:21.273Z" },
+ { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873, upload-time = "2024-08-06T20:32:25.131Z" },
+ { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302, upload-time = "2024-08-06T20:32:26.511Z" },
+ { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154, upload-time = "2024-08-06T20:32:28.363Z" },
+ { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223, upload-time = "2024-08-06T20:32:30.058Z" },
+ { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542, upload-time = "2024-08-06T20:32:31.881Z" },
+ { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164, upload-time = "2024-08-06T20:32:37.083Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611, upload-time = "2024-08-06T20:32:38.898Z" },
+ { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591, upload-time = "2024-08-06T20:32:40.241Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338, upload-time = "2024-08-06T20:32:41.93Z" },
+ { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" },
+ { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" },
+ { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" },
+ { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" },
+ { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" },
+ { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" },
+]
+
+[[package]]
+name = "six"
+version = "1.17.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" },
+]
+
+[[package]]
+name = "types-protobuf"
+version = "6.30.2.20250914"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/36/d1/e12dad323fe6e2455b768828de288f60d5160f41dad5d31af8ef92a6acbb/types_protobuf-6.30.2.20250914.tar.gz", hash = "sha256:c2105326d0a52de3d33b84af0010d834ebbd4c17c50ff261fa82551ab75d9559", size = 62424, upload-time = "2025-09-14T02:56:00.798Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/6c/c4/3fcb1f8e03456a8a33a5dfb9f9788b0a91023e5fad6a37d46fc6831629a7/types_protobuf-6.30.2.20250914-py3-none-any.whl", hash = "sha256:cfc24977c0f38cf2896d918a59faed7650eb983be6070343a6204ac8ac0a297e", size = 76546, upload-time = "2025-09-14T02:55:59.489Z" },
+]
+
+[[package]]
+name = "types-python-dateutil"
+version = "2.9.0.20250822"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/0c/0a/775f8551665992204c756be326f3575abba58c4a3a52eef9909ef4536428/types_python_dateutil-2.9.0.20250822.tar.gz", hash = "sha256:84c92c34bd8e68b117bff742bc00b692a1e8531262d4507b33afcc9f7716cd53", size = 16084, upload-time = "2025-08-22T03:02:00.613Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ab/d9/a29dfa84363e88b053bf85a8b7f212a04f0d7343a4d24933baa45c06e08b/types_python_dateutil-2.9.0.20250822-py3-none-any.whl", hash = "sha256:849d52b737e10a6dc6621d2bd7940ec7c65fcb69e6aa2882acf4e56b2b508ddc", size = 17892, upload-time = "2025-08-22T03:01:59.436Z" },
+]
+
+[[package]]
+name = "types-pyyaml"
+version = "6.0.12.20250915"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/7e/69/3c51b36d04da19b92f9e815be12753125bd8bc247ba0470a982e6979e71c/types_pyyaml-6.0.12.20250915.tar.gz", hash = "sha256:0f8b54a528c303f0e6f7165687dd33fafa81c807fcac23f632b63aa624ced1d3", size = 17522, upload-time = "2025-09-15T03:01:00.728Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/bd/e0/1eed384f02555dde685fff1a1ac805c1c7dcb6dd019c916fe659b1c1f9ec/types_pyyaml-6.0.12.20250915-py3-none-any.whl", hash = "sha256:e7d4d9e064e89a3b3cae120b4990cd370874d2bf12fa5f46c97018dd5d3c9ab6", size = 20338, upload-time = "2025-09-15T03:00:59.218Z" },
+]
diff --git a/flyteidl2/gen_utils/rust/Cargo.toml b/flyteidl2/gen_utils/rust/Cargo.toml
new file mode 100644
index 00000000000..8c17d109ab3
--- /dev/null
+++ b/flyteidl2/gen_utils/rust/Cargo.toml
@@ -0,0 +1,50 @@
+[package]
+name = "flyteidl2"
+version = "0.1.0"
+edition = "2021"
+description = "Rust bindings and utilities for FlyteIDL protobufs."
+license = "Apache-2.0"
+
+[dependencies]
+pyo3 = { version = "0.24", features = ["experimental-async"] }
+pyo3-async-runtimes = { version = "0.24", features = ["tokio-runtime"] }
+tokio = { version = "1.0", features = ["full"] }
+tonic = "0.12.3"
+prost = { version = "0.13.5", features = ["std"] }
+prost-types = { version = "0.12", features = ["std"] }
+futures = "0.3"
+tower = "0.4"
+tower-http = { version = "0.5", features = ["trace"] }
+tracing = "0.1"
+tracing-subscriber = "0.3"
+async-trait = "0.1"
+thiserror = "1.0"
+serde = { version = "1.0", features = ["derive"] }
+pbjson = { version = "0.7.0" }
+pbjson-types = { version = "0.7.0" }
+protobuf = { version = "4.31.1-release" }
+prost-build = "0.14"
+quote = "1.0"
+syn = { version = "2.0", features = ["full", "extra-traits"] }
+prettyplease = "0.2"
+regex = "1.11.1"
+
+[lib]
+name = "flyteidl2"
+path = "src/lib.rs"
+#crate-type = ["cdylib"]
+
+[features]
+default = []
+## @@protoc_insertion_point(features)
+
+[build-dependencies]
+prost-build = "0.14"
+pbjson-build = { version = "0.7.0" }
+quote = "1.0"
+syn = { version = "2.0", features = ["full", "extra-traits"] }
+prettyplease = "0.2"
+regex = "1.10"
+
+#[profile.dev.build-override]
+#debug = true
\ No newline at end of file
diff --git a/flyteidl2/gen_utils/rust/build.rs b/flyteidl2/gen_utils/rust/build.rs
new file mode 100644
index 00000000000..10e4c5c9df8
--- /dev/null
+++ b/flyteidl2/gen_utils/rust/build.rs
@@ -0,0 +1,516 @@
+//! Compiles Protocol Buffers and FlatBuffers schema definitions into
+//! native Rust types.
+
+use prettyplease::unparse;
+use quote::quote;
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+use syn::{parse_quote, Type};
+
+type Error = Box;
+type Result = std::result::Result;
+
+fn generate_boxed_impls() -> String {
+ // Define the types we want to implement Box for
+ let types: Vec = vec![
+ parse_quote!(crate::validate::FieldRules),
+ parse_quote!(crate::validate::RepeatedRules),
+ parse_quote!(crate::validate::MapRules),
+ parse_quote!(crate::flyteidl::core::LiteralType),
+ parse_quote!(crate::flyteidl::core::Literal),
+ parse_quote!(crate::flyteidl::core::Union),
+ parse_quote!(crate::google::protobuf::FeatureSet),
+ parse_quote!(crate::flyteidl::core::Scalar),
+ // parse_quote!(crate::pyo3::types::PyBytes),
+ ];
+
+ // Generate the code using quote
+ let tokens = quote! {
+ use pyo3::prelude::*;
+ use pyo3::conversion::{IntoPyObject, FromPyObject};
+ use std::convert::Infallible;
+
+ #(
+ impl<'py> IntoPyObject<'py> for Box<#types> {
+ type Target = PyAny;
+ type Output = Bound<'py, Self::Target>;
+ type Error = Infallible;
+
+ fn into_pyobject(self, py: Python<'py>) -> Result {
+ Ok(self.as_ref().clone().into_py(py).into_bound(py))
+ }
+ }
+
+ impl<'py> FromPyObject<'py> for Box<#types> {
+ fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult {
+ Ok(Box::new(#types::extract_bound(obj)?))
+ }
+ }
+ )*
+ };
+
+ // Convert the generated code to a string and format it
+ let syntax = syn::parse_file(&tokens.to_string()).expect("Failed to parse generated code");
+ unparse(&syntax)
+}
+
+// Map file names to module paths
+fn file_to_modpath(file: &str) -> Option<&'static str> {
+ match file {
+ "cloudidl.common.rs" => Some("crate::cloudidl::common"),
+ "cloudidl.workflow.rs" => Some("crate::cloudidl::workflow"),
+ "cloudidl.logs.dataplane.rs" => Some("crate::cloudidl::logs::dataplane"),
+ "flyteidl.core.rs" => Some("crate::flyteidl::core"),
+ "google.rpc.rs" => Some("crate::google::rpc"),
+ "validate.rs" => Some("crate::validate"),
+ _ => None,
+ }
+}
+
+#[derive(Debug)]
+struct ProstStructInfo {
+ ty: Type,
+ fq_name: String,
+ fields: Vec<(String, String)>,
+ is_tuple: bool,
+ mod_path: String, // e.g. crate::cloudidl::common
+}
+
+// Find all prost-generated types in src/
+fn find_prost_types(src_dir: &PathBuf) -> Vec {
+ use std::collections::HashSet;
+ use std::path::Path;
+ use syn::{Fields, Item};
+
+ let mut structs = Vec::new();
+ let mut visited = HashSet::new();
+
+ // Helper to recursively walk modules
+ fn walk_items(items: &[Item], mod_path: &mut Vec, structs: &mut Vec) {
+ for item in items {
+ match item {
+ Item::Mod(m) => {
+ mod_path.push(m.ident.to_string());
+ // If inline module, recurse into its content
+ if let Some((_, items)) = &m.content {
+ walk_items(items, mod_path, structs);
+ }
+ // Outline modules are handled in the main loop
+ mod_path.pop();
+ }
+ Item::Struct(s) => {
+ let fq = format!("{}::{}", mod_path.join("::"), s.ident);
+ let ty = match syn::parse_str::(&fq) {
+ Ok(t) => t,
+ Err(_) => continue,
+ };
+ let (fields, is_tuple) = match &s.fields {
+ Fields::Named(fields) => {
+ let fields = fields
+ .named
+ .iter()
+ .map(|f| {
+ let name = f.ident.as_ref().unwrap().to_string();
+ let ty = quote::ToTokens::to_token_stream(&f.ty).to_string();
+ (name, ty)
+ })
+ .collect();
+ (fields, false)
+ }
+ Fields::Unnamed(fields) => {
+ let fields = fields
+ .unnamed
+ .iter()
+ .enumerate()
+ .map(|(i, f)| {
+ let name = format!("field{}", i);
+ let ty = quote::ToTokens::to_token_stream(&f.ty).to_string();
+ (name, ty)
+ })
+ .collect();
+ (fields, true)
+ }
+ Fields::Unit => (vec![], false),
+ };
+ let mod_path = mod_path.join("::");
+ let mod_path = if !mod_path.is_empty() {
+ format!("crate::{}", mod_path)
+ } else {
+ "crate".to_string()
+ };
+ structs.push(ProstStructInfo {
+ ty,
+ fq_name: fq,
+ fields,
+ is_tuple,
+ mod_path,
+ });
+ }
+ _ => {}
+ }
+ }
+ }
+
+ // Helper to parse a file and walk its items
+ fn parse_and_walk(
+ path: &Path,
+ mod_path: &mut Vec,
+ structs: &mut Vec,
+ visited: &mut HashSet,
+ ) {
+ if !visited.insert(path.to_path_buf()) {
+ return;
+ }
+ let content = match std::fs::read_to_string(path) {
+ Ok(c) => c,
+ Err(_) => return,
+ };
+ let ast: syn::File = match syn::parse_file(&content) {
+ Ok(f) => f,
+ Err(_) => return,
+ };
+ walk_items(&ast.items, mod_path, structs);
+ // Recursively handle outline modules
+ for item in &ast.items {
+ if let Item::Mod(m) = item {
+ if m.content.is_none() {
+ // Outline module: look for mod.rs or .rs
+ let mod_name = m.ident.to_string();
+ let parent = path.parent().unwrap_or(Path::new(""));
+ let mod_rs = parent.join(&mod_name).join("mod.rs");
+ let mod_file = parent.join(format!("{}.rs", mod_name));
+ if mod_rs.exists() {
+ mod_path.push(mod_name.clone());
+ parse_and_walk(&mod_rs, mod_path, structs, visited);
+ mod_path.pop();
+ } else if mod_file.exists() {
+ mod_path.push(mod_name.clone());
+ parse_and_walk(&mod_file, mod_path, structs, visited);
+ mod_path.pop();
+ }
+ }
+ }
+ }
+ }
+
+ for entry in std::fs::read_dir(src_dir).unwrap() {
+ let entry = entry.unwrap();
+ let path = entry.path();
+ let file_name = path.file_name().unwrap().to_string_lossy();
+
+ if !file_name.ends_with(".rs") || file_name == "serde_impl.rs" || file_name == "lib.rs" {
+ continue;
+ }
+
+ let modpath = match file_to_modpath(&file_name) {
+ Some(m) => m.replace("crate::", ""),
+ None => continue,
+ };
+ let mut mod_path: Vec = modpath.split("::").map(|s| s.to_string()).collect();
+ parse_and_walk(&path, &mut mod_path, &mut structs, &mut visited);
+ }
+ structs
+}
+
+fn generate_encode_decode(infos: &[ProstStructInfo]) -> String {
+ // Helper to qualify types
+ fn qualify_type(ty: &str, current_mod: &str) -> String {
+ let primitives = [
+ "String", "u8", "u16", "u32", "u64", "i8", "i16", "i32", "i64", "bool", "f32", "f64",
+ "usize", "isize", "char", "str",
+ ];
+ let ty = ty.trim();
+ let mut ty_clean = ty
+ .replace(" :: ", "::")
+ .replace("crate ::", "crate::")
+ .replace("super ::", "super::")
+ .replace("self ::", "self::");
+ while ty_clean.contains("::::") {
+ ty_clean = ty_clean.replace("::::", "::");
+ }
+ // Handle generics recursively (e.g., Option, Vec)
+ if let Some(start) = ty_clean.find('<') {
+ if let Some(end) = ty_clean.rfind('>') {
+ let outer = &ty_clean[..start];
+ let inner = &ty_clean[start + 1..end];
+ let qualified_inner = inner
+ .split(',')
+ .map(|s| qualify_type(s, current_mod))
+ .collect::>()
+ .join(", ");
+ return format!("{}<{}>", outer, qualified_inner);
+ }
+ }
+ // Handle super:: prefix by resolving up the module path
+ if ty_clean.starts_with("super::") {
+ let mut mod_parts: Vec<&str> = current_mod.split("::").collect();
+ let mut ty_remainder = ty_clean.to_string();
+ while ty_remainder.starts_with("super::") {
+ ty_remainder = ty_remainder[7..].to_string();
+ if mod_parts.len() > 1 {
+ mod_parts.pop();
+ }
+ }
+ let abs_path = if ty_remainder.is_empty() {
+ mod_parts.join("::")
+ } else {
+ format!("{}::{}", mod_parts.join("::"), ty_remainder)
+ };
+ return abs_path;
+ }
+ if ty_clean.starts_with("self::") {
+ // self:: just means current_mod
+ let ty_remainder = &ty_clean[6..];
+ return format!("{}::{}", current_mod, ty_remainder);
+ }
+ if ty_clean.starts_with("crate::")
+ || ty_clean.starts_with("::")
+ || ty_clean.starts_with("prost::")
+ || ty_clean.starts_with("core::")
+ || ty_clean.starts_with("std::")
+ || ty_clean.starts_with("alloc::")
+ {
+ ty_clean
+ } else if primitives.iter().any(|&p| ty_clean == p)
+ || ty_clean.starts_with("Option")
+ || ty_clean.starts_with("Vec")
+ || ty_clean.starts_with("HashMap")
+ {
+ ty_clean
+ } else if ty_clean.contains("::") {
+ // Prepend current_mod for relative paths like enriched_identity::Principal
+ format!("{}::{}", current_mod, ty_clean)
+ } else {
+ format!("{}::{}", current_mod, ty_clean.replace(' ', ""))
+ }
+ }
+ let mut py_methods = String::new();
+ for info in infos {
+ let fq = info
+ .fq_name
+ .replace("crate :: ", "crate::")
+ .replace(" :: ", "::");
+ let fields = &info.fields;
+ let is_tuple = info.is_tuple;
+ let mod_path = &info.mod_path;
+ let py_new = if fields.is_empty() {
+ format!(
+ " #[new]\n pub fn py_new() -> Self {{\n Self::default()\n }}\n"
+ )
+ } else if is_tuple {
+ let args = fields
+ .iter()
+ .map(|(n, t)| format!("{}: {}", n, qualify_type(t, mod_path)))
+ .collect::>()
+ .join(", ");
+ let vals = fields
+ .iter()
+ .map(|(n, _)| n.clone())
+ .collect::>()
+ .join(", ");
+ format!(
+ " #[new]\n pub fn py_new({}) -> Self {{\n Self({})\n }}\n",
+ args, vals
+ )
+ } else {
+ let args = fields
+ .iter()
+ .map(|(n, t)| format!("{}: {}", n, qualify_type(t, mod_path)))
+ .collect::>()
+ .join(", ");
+ let vals = fields
+ .iter()
+ .map(|(n, _)| format!("{}", n))
+ .collect::>()
+ .join(", ");
+ format!(
+ " #[new]\n pub fn py_new({}) -> Self {{\n Self {{ {} }}\n }}\n",
+ args, vals
+ )
+ };
+ py_methods += &format!(
+ r#"
+ #[::pyo3::pymethods]
+ impl {fq} {{
+{py_new}
+ fn __repr__(&self) -> String {{
+ format!("{{:?}}", self)
+ }}
+ fn __str__(&self) -> String {{
+ format!("{{:?}}", self)
+ }}
+ }}
+"#,
+ fq = fq,
+ py_new = py_new
+ );
+ }
+
+ let syntax = syn::parse_file(&py_methods.to_string()).expect("Failed to parse generated code");
+ unparse(&syntax)
+}
+
+#[derive(Default)]
+struct ModuleNode {
+ children: std::collections::HashMap,
+ types: Vec, // store fully qualified type paths
+}
+
+fn build_module_tree(
+ types: &[Type],
+ all_types: &mut std::collections::BTreeSet,
+) -> ModuleNode {
+ let mut root = ModuleNode::default();
+ for ty in types {
+ let fq = quote!(#ty)
+ .to_string()
+ .replace("crate :: ", "crate::")
+ .replace(" :: ", "::");
+ let parts: Vec<_> = fq.split("::").map(|s| s.trim().to_string()).collect();
+ if parts.len() < 2 {
+ continue;
+ }
+ let (mod_path, type_name) = parts.split_at(parts.len() - 1);
+ let mut node = &mut root;
+ for part in mod_path {
+ node = node.children.entry(part.clone()).or_default();
+ }
+ let full_type = parts.join("::");
+ node.types.push(full_type.clone());
+ all_types.insert(full_type);
+ }
+ root
+}
+
+fn generate_pymodules_file(
+ module_tree: &ModuleNode,
+ all_types: &std::collections::BTreeSet,
+) -> String {
+ let mut code = String::new();
+ code += "use pyo3::prelude::*;\n\
+ #[pyo3::pymodule]\n";
+ // for ty in all_types {
+ // code += &format!(
+ // "use {};
+ // ",
+ // ty
+ // );
+ // }
+ code += &generate_pymodules(module_tree, &[]);
+ code
+}
+
+fn generate_pymodules(node: &ModuleNode, mod_path: &[String]) -> String {
+ let mod_name = mod_path
+ .last()
+ .cloned()
+ .unwrap_or_else(|| "cloud".to_string());
+ let func_name = if mod_path.is_empty() {
+ "init_pymodule".to_string()
+ } else {
+ format!("init_{}", mod_path.join("_"))
+ };
+ let mut code = format!(
+ "pub fn {}_mod(_py: pyo3::Python, m: &Bound<'_, PyModule>) -> pyo3::PyResult<()> {{\n",
+ mod_name
+ );
+ for ty in &node.types {
+ code += &format!(" m.add_class::()?;\n", ty);
+ }
+ for (child_name, _child_node) in &node.children {
+ let submod_func = format!("{}_mod", child_name.clone());
+ code += &format!(
+ " let submod = pyo3::types::PyModule::new(_py, \"{}\")?;\n",
+ child_name
+ );
+ code += &format!(" {}(_py, &submod)?;\n", submod_func);
+ code += &format!(" m.add_submodule(&submod)?;\n");
+ }
+ code += " Ok(())\n}\n";
+ for (child_name, child_node) in &node.children {
+ let mut child_path = mod_path.to_vec();
+ child_path.push(child_name.clone());
+ code += &generate_pymodules(child_node, &child_path);
+ }
+ code
+}
+
+fn main() -> Result<()> {
+ let descriptor_path: PathBuf = "descriptors.bin".into();
+ println!("cargo:rerun-if-changed={}", descriptor_path.display());
+
+ let mut config = prost_build::Config::new();
+ config
+ .file_descriptor_set_path(&descriptor_path)
+ .compile_well_known_types()
+ .disable_comments(["."])
+ // .bytes(["."]) // Add prost::bytes::Bytes exclusion
+ .type_attribute(".", "#[pyo3::pyclass(dict, get_all, set_all)]")
+ // .type_attribute(".", "#[pyo3_prost::pyclass_for_prost_struct]")
+ // .type_attribute("bytes::Bytes", "#[pyo3::pyclass(dict, get_all, set_all)]")
+ // .type_attribute(
+ // "::pyo3::types::PyBytes",
+ // "#[pyo3::pyclass(dict, get_all, set_all)]",
+ // )
+ .skip_protoc_run();
+
+ let empty: &[&str] = &[];
+ config.compile_protos(empty, empty)?;
+
+ let descriptor_set = std::fs::read(descriptor_path)?;
+ pbjson_build::Builder::new()
+ .register_descriptors(&descriptor_set)?
+ .exclude([
+ ".google.protobuf.Duration",
+ ".google.protobuf.Timestamp",
+ ".google.protobuf.Value",
+ ".google.protobuf.Struct",
+ ".google.protobuf.ListValue",
+ ".google.protobuf.NullValue",
+ ".google.protobuf.BoolValue",
+ ".google.protobuf.BytesValue",
+ ".google.protobuf.DoubleValue",
+ ".google.protobuf.FloatValue",
+ ".google.protobuf.Int32Value",
+ ".google.protobuf.Int64Value",
+ ".google.protobuf.StringValue",
+ ".google.protobuf.UInt32Value",
+ ".google.protobuf.UInt64Value",
+ ])
+ .build(&[".google"])?;
+
+ // Generate Box implementations
+ let boxed_impls = generate_boxed_impls();
+ let out_dir = std::env::var("OUT_DIR").unwrap();
+ let boxed_impls_path = PathBuf::from(&out_dir).join("boxed_impls.rs");
+ let mut file = File::create(boxed_impls_path)?;
+ file.write_all(boxed_impls.as_bytes())?;
+
+ // Find all prost types
+ let src_dir = PathBuf::from("src");
+ let prost_infos = find_prost_types(&src_dir);
+ eprintln!("Found {} prost types", prost_infos.len());
+ eprintln!("Generating implementations for {:?} types", prost_infos);
+ // unsafe { std::intrinsics::breakpoint(); }
+
+ // Generate encode,decode implementations for all prost types
+ let serde_impls = generate_encode_decode(&prost_infos);
+ let serde_impls_path = PathBuf::from(&out_dir).join("serde_impls.rs");
+ let mut file = File::create(serde_impls_path)?;
+ file.write_all(serde_impls.as_bytes())?;
+
+ // --- New: Generate PyO3 module tree code ---
+ let mut all_types = std::collections::BTreeSet::new();
+ let module_tree = build_module_tree(
+ &prost_infos.iter().map(|i| i.ty.clone()).collect::>(),
+ &mut all_types,
+ );
+ let pymodules_code = generate_pymodules_file(&module_tree, &all_types);
+ let pymodules_path = PathBuf::from(&out_dir).join("pymodules.rs");
+ let mut file = File::create(pymodules_path)?;
+ file.write_all(pymodules_code.as_bytes())?;
+ // --- End new ---
+
+ Ok(())
+}
diff --git a/flyteidl2/gen_utils/rust/descriptors.bin b/flyteidl2/gen_utils/rust/descriptors.bin
new file mode 100644
index 00000000000..0e370ea3c1b
Binary files /dev/null and b/flyteidl2/gen_utils/rust/descriptors.bin differ
diff --git a/flyteidl2/gen_utils/rust/pyproject.toml b/flyteidl2/gen_utils/rust/pyproject.toml
new file mode 100644
index 00000000000..350cb34e257
--- /dev/null
+++ b/flyteidl2/gen_utils/rust/pyproject.toml
@@ -0,0 +1,17 @@
+[build-system]
+requires = ["maturin>=1.9,<2.0"]
+build-backend = "maturin"
+
+[project]
+name = "flyteidl"
+version = "0.1.0"
+description = "Rust protos for Flyte"
+requires-python = ">=3.10"
+classifiers = [
+ "Programming Language :: Python",
+ "Programming Language :: Rust",
+]
+
+[tool.maturin]
+python-source = "src"
+module-name = "flyte_mod"
diff --git a/flyteidl2/gen_utils/rust/src/google.rpc.rs b/flyteidl2/gen_utils/rust/src/google.rpc.rs
new file mode 100644
index 00000000000..2576dadaf8a
--- /dev/null
+++ b/flyteidl2/gen_utils/rust/src/google.rpc.rs
@@ -0,0 +1,163 @@
+// @generated
+// This file is @generated by prost-build.
+/// The `Status` type defines a logical error model that is suitable for
+/// different programming environments, including REST APIs and RPC APIs. It is
+/// used by [gRPC](). Each `Status` message contains
+/// three pieces of data: error code, error message, and error details.
+///
+/// You can find out more about this error model and how to work with it in the
+/// [API Design Guide]().
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct Status {
+ /// The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code].
+ #[prost(int32, tag="1")]
+ pub code: i32,
+ /// A developer-facing error message, which should be in English. Any
+ /// user-facing error message should be localized and sent in the
+ /// [google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client.
+ #[prost(string, tag="2")]
+ pub message: ::prost::alloc::string::String,
+ /// A list of messages that carry the error details. There is a common set of
+ /// message types for APIs to use.
+ #[prost(message, repeated, tag="3")]
+ pub details: ::prost::alloc::vec::Vec,
+}
+/// Encoded file descriptor set for the `google.rpc` package
+pub const FILE_DESCRIPTOR_SET: &[u8] = &[
+ 0x0a, 0xc5, 0x10, 0x0a, 0x17, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x72, 0x70, 0x63, 0x2f,
+ 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x67, 0x6f,
+ 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x72, 0x70, 0x63, 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
+ 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72,
+ 0x6f, 0x74, 0x6f, 0x22, 0x66, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a,
+ 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x63, 0x6f, 0x64,
+ 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01,
+ 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2e, 0x0a, 0x07, 0x64,
+ 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67,
+ 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41,
+ 0x6e, 0x79, 0x52, 0x07, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x42, 0xa2, 0x01, 0x0a, 0x0e,
+ 0x63, 0x6f, 0x6d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x72, 0x70, 0x63, 0x42, 0x0b,
+ 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x37, 0x67,
+ 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67,
+ 0x2f, 0x67, 0x65, 0x6e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
+ 0x61, 0x70, 0x69, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x3b,
+ 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0xf8, 0x01, 0x01, 0xa2, 0x02, 0x03, 0x47, 0x52, 0x58, 0xaa,
+ 0x02, 0x0a, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x52, 0x70, 0x63, 0xca, 0x02, 0x0a, 0x47,
+ 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x5c, 0x52, 0x70, 0x63, 0xe2, 0x02, 0x16, 0x47, 0x6f, 0x6f, 0x67,
+ 0x6c, 0x65, 0x5c, 0x52, 0x70, 0x63, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61,
+ 0x74, 0x61, 0xea, 0x02, 0x0b, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x3a, 0x3a, 0x52, 0x70, 0x63,
+ 0x4a, 0xed, 0x0d, 0x0a, 0x06, 0x12, 0x04, 0x0e, 0x00, 0x2e, 0x01, 0x0a, 0xbc, 0x04, 0x0a, 0x01,
+ 0x0c, 0x12, 0x03, 0x0e, 0x00, 0x12, 0x32, 0xb1, 0x04, 0x20, 0x43, 0x6f, 0x70, 0x79, 0x72, 0x69,
+ 0x67, 0x68, 0x74, 0x20, 0x32, 0x30, 0x32, 0x30, 0x20, 0x47, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x20,
+ 0x4c, 0x4c, 0x43, 0x0a, 0x0a, 0x20, 0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x64, 0x20, 0x75,
+ 0x6e, 0x64, 0x65, 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x41, 0x70, 0x61, 0x63, 0x68, 0x65, 0x20,
+ 0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x2c, 0x20, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
+ 0x20, 0x32, 0x2e, 0x30, 0x20, 0x28, 0x74, 0x68, 0x65, 0x20, 0x22, 0x4c, 0x69, 0x63, 0x65, 0x6e,
+ 0x73, 0x65, 0x22, 0x29, 0x3b, 0x0a, 0x20, 0x79, 0x6f, 0x75, 0x20, 0x6d, 0x61, 0x79, 0x20, 0x6e,
+ 0x6f, 0x74, 0x20, 0x75, 0x73, 0x65, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x66, 0x69, 0x6c, 0x65,
+ 0x20, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, 0x20, 0x69, 0x6e, 0x20, 0x63, 0x6f, 0x6d, 0x70, 0x6c,
+ 0x69, 0x61, 0x6e, 0x63, 0x65, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x74, 0x68, 0x65, 0x20, 0x4c,
+ 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x2e, 0x0a, 0x20, 0x59, 0x6f, 0x75, 0x20, 0x6d, 0x61, 0x79,
+ 0x20, 0x6f, 0x62, 0x74, 0x61, 0x69, 0x6e, 0x20, 0x61, 0x20, 0x63, 0x6f, 0x70, 0x79, 0x20, 0x6f,
+ 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x20, 0x61, 0x74,
+ 0x0a, 0x0a, 0x20, 0x20, 0x20, 0x20, 0x20, 0x68, 0x74, 0x74, 0x70, 0x3a, 0x2f, 0x2f, 0x77, 0x77,
+ 0x77, 0x2e, 0x61, 0x70, 0x61, 0x63, 0x68, 0x65, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x6c, 0x69, 0x63,
+ 0x65, 0x6e, 0x73, 0x65, 0x73, 0x2f, 0x4c, 0x49, 0x43, 0x45, 0x4e, 0x53, 0x45, 0x2d, 0x32, 0x2e,
+ 0x30, 0x0a, 0x0a, 0x20, 0x55, 0x6e, 0x6c, 0x65, 0x73, 0x73, 0x20, 0x72, 0x65, 0x71, 0x75, 0x69,
+ 0x72, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x62, 0x6c,
+ 0x65, 0x20, 0x6c, 0x61, 0x77, 0x20, 0x6f, 0x72, 0x20, 0x61, 0x67, 0x72, 0x65, 0x65, 0x64, 0x20,
+ 0x74, 0x6f, 0x20, 0x69, 0x6e, 0x20, 0x77, 0x72, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x2c, 0x20, 0x73,
+ 0x6f, 0x66, 0x74, 0x77, 0x61, 0x72, 0x65, 0x0a, 0x20, 0x64, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62,
+ 0x75, 0x74, 0x65, 0x64, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x4c,
+ 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x20, 0x69, 0x73, 0x20, 0x64, 0x69, 0x73, 0x74, 0x72, 0x69,
+ 0x62, 0x75, 0x74, 0x65, 0x64, 0x20, 0x6f, 0x6e, 0x20, 0x61, 0x6e, 0x20, 0x22, 0x41, 0x53, 0x20,
+ 0x49, 0x53, 0x22, 0x20, 0x42, 0x41, 0x53, 0x49, 0x53, 0x2c, 0x0a, 0x20, 0x57, 0x49, 0x54, 0x48,
+ 0x4f, 0x55, 0x54, 0x20, 0x57, 0x41, 0x52, 0x52, 0x41, 0x4e, 0x54, 0x49, 0x45, 0x53, 0x20, 0x4f,
+ 0x52, 0x20, 0x43, 0x4f, 0x4e, 0x44, 0x49, 0x54, 0x49, 0x4f, 0x4e, 0x53, 0x20, 0x4f, 0x46, 0x20,
+ 0x41, 0x4e, 0x59, 0x20, 0x4b, 0x49, 0x4e, 0x44, 0x2c, 0x20, 0x65, 0x69, 0x74, 0x68, 0x65, 0x72,
+ 0x20, 0x65, 0x78, 0x70, 0x72, 0x65, 0x73, 0x73, 0x20, 0x6f, 0x72, 0x20, 0x69, 0x6d, 0x70, 0x6c,
+ 0x69, 0x65, 0x64, 0x2e, 0x0a, 0x20, 0x53, 0x65, 0x65, 0x20, 0x74, 0x68, 0x65, 0x20, 0x4c, 0x69,
+ 0x63, 0x65, 0x6e, 0x73, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x74, 0x68, 0x65, 0x20, 0x73, 0x70,
+ 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, 0x20, 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, 0x20,
+ 0x67, 0x6f, 0x76, 0x65, 0x72, 0x6e, 0x69, 0x6e, 0x67, 0x20, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73,
+ 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x0a, 0x20, 0x6c, 0x69, 0x6d, 0x69, 0x74,
+ 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x75, 0x6e, 0x64, 0x65, 0x72, 0x20, 0x74, 0x68, 0x65,
+ 0x20, 0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x2e, 0x0a, 0x0a, 0x08, 0x0a, 0x01, 0x02, 0x12,
+ 0x03, 0x10, 0x00, 0x13, 0x0a, 0x09, 0x0a, 0x02, 0x03, 0x00, 0x12, 0x03, 0x12, 0x00, 0x23, 0x0a,
+ 0x08, 0x0a, 0x01, 0x08, 0x12, 0x03, 0x14, 0x00, 0x1f, 0x0a, 0x09, 0x0a, 0x02, 0x08, 0x1f, 0x12,
+ 0x03, 0x14, 0x00, 0x1f, 0x0a, 0x08, 0x0a, 0x01, 0x08, 0x12, 0x03, 0x15, 0x00, 0x4e, 0x0a, 0x09,
+ 0x0a, 0x02, 0x08, 0x0b, 0x12, 0x03, 0x15, 0x00, 0x4e, 0x0a, 0x08, 0x0a, 0x01, 0x08, 0x12, 0x03,
+ 0x16, 0x00, 0x22, 0x0a, 0x09, 0x0a, 0x02, 0x08, 0x0a, 0x12, 0x03, 0x16, 0x00, 0x22, 0x0a, 0x08,
+ 0x0a, 0x01, 0x08, 0x12, 0x03, 0x17, 0x00, 0x2c, 0x0a, 0x09, 0x0a, 0x02, 0x08, 0x08, 0x12, 0x03,
+ 0x17, 0x00, 0x2c, 0x0a, 0x08, 0x0a, 0x01, 0x08, 0x12, 0x03, 0x18, 0x00, 0x27, 0x0a, 0x09, 0x0a,
+ 0x02, 0x08, 0x01, 0x12, 0x03, 0x18, 0x00, 0x27, 0x0a, 0xbe, 0x03, 0x0a, 0x02, 0x04, 0x00, 0x12,
+ 0x04, 0x22, 0x00, 0x2e, 0x01, 0x1a, 0xb1, 0x03, 0x20, 0x54, 0x68, 0x65, 0x20, 0x60, 0x53, 0x74,
+ 0x61, 0x74, 0x75, 0x73, 0x60, 0x20, 0x74, 0x79, 0x70, 0x65, 0x20, 0x64, 0x65, 0x66, 0x69, 0x6e,
+ 0x65, 0x73, 0x20, 0x61, 0x20, 0x6c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x20, 0x65, 0x72, 0x72,
+ 0x6f, 0x72, 0x20, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x69, 0x73,
+ 0x20, 0x73, 0x75, 0x69, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x0a, 0x20, 0x64,
+ 0x69, 0x66, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x74, 0x20, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x61, 0x6d,
+ 0x6d, 0x69, 0x6e, 0x67, 0x20, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74,
+ 0x73, 0x2c, 0x20, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x69, 0x6e, 0x67, 0x20, 0x52, 0x45, 0x53,
+ 0x54, 0x20, 0x41, 0x50, 0x49, 0x73, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x52, 0x50, 0x43, 0x20, 0x41,
+ 0x50, 0x49, 0x73, 0x2e, 0x20, 0x49, 0x74, 0x20, 0x69, 0x73, 0x0a, 0x20, 0x75, 0x73, 0x65, 0x64,
+ 0x20, 0x62, 0x79, 0x20, 0x5b, 0x67, 0x52, 0x50, 0x43, 0x5d, 0x28, 0x68, 0x74, 0x74, 0x70, 0x73,
+ 0x3a, 0x2f, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x72,
+ 0x70, 0x63, 0x29, 0x2e, 0x20, 0x45, 0x61, 0x63, 0x68, 0x20, 0x60, 0x53, 0x74, 0x61, 0x74, 0x75,
+ 0x73, 0x60, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x20, 0x63, 0x6f, 0x6e, 0x74, 0x61,
+ 0x69, 0x6e, 0x73, 0x0a, 0x20, 0x74, 0x68, 0x72, 0x65, 0x65, 0x20, 0x70, 0x69, 0x65, 0x63, 0x65,
+ 0x73, 0x20, 0x6f, 0x66, 0x20, 0x64, 0x61, 0x74, 0x61, 0x3a, 0x20, 0x65, 0x72, 0x72, 0x6f, 0x72,
+ 0x20, 0x63, 0x6f, 0x64, 0x65, 0x2c, 0x20, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x20, 0x6d, 0x65, 0x73,
+ 0x73, 0x61, 0x67, 0x65, 0x2c, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x20,
+ 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x0a, 0x0a, 0x20, 0x59, 0x6f, 0x75, 0x20, 0x63,
+ 0x61, 0x6e, 0x20, 0x66, 0x69, 0x6e, 0x64, 0x20, 0x6f, 0x75, 0x74, 0x20, 0x6d, 0x6f, 0x72, 0x65,
+ 0x20, 0x61, 0x62, 0x6f, 0x75, 0x74, 0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x65, 0x72, 0x72, 0x6f,
+ 0x72, 0x20, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x68, 0x6f, 0x77, 0x20,
+ 0x74, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6b, 0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x69, 0x74, 0x20,
+ 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65, 0x0a, 0x20, 0x5b, 0x41, 0x50, 0x49, 0x20, 0x44, 0x65, 0x73,
+ 0x69, 0x67, 0x6e, 0x20, 0x47, 0x75, 0x69, 0x64, 0x65, 0x5d, 0x28, 0x68, 0x74, 0x74, 0x70, 0x73,
+ 0x3a, 0x2f, 0x2f, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
+ 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x64, 0x65, 0x73, 0x69, 0x67, 0x6e, 0x2f,
+ 0x65, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x29, 0x2e, 0x0a, 0x0a, 0x0a, 0x0a, 0x03, 0x04, 0x00, 0x01,
+ 0x12, 0x03, 0x22, 0x08, 0x0e, 0x0a, 0x64, 0x0a, 0x04, 0x04, 0x00, 0x02, 0x00, 0x12, 0x03, 0x24,
+ 0x02, 0x11, 0x1a, 0x57, 0x20, 0x54, 0x68, 0x65, 0x20, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x20,
+ 0x63, 0x6f, 0x64, 0x65, 0x2c, 0x20, 0x77, 0x68, 0x69, 0x63, 0x68, 0x20, 0x73, 0x68, 0x6f, 0x75,
+ 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x61, 0x6e, 0x20, 0x65, 0x6e, 0x75, 0x6d, 0x20, 0x76, 0x61,
+ 0x6c, 0x75, 0x65, 0x20, 0x6f, 0x66, 0x20, 0x5b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x72,
+ 0x70, 0x63, 0x2e, 0x43, 0x6f, 0x64, 0x65, 0x5d, 0x5b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e,
+ 0x72, 0x70, 0x63, 0x2e, 0x43, 0x6f, 0x64, 0x65, 0x5d, 0x2e, 0x0a, 0x0a, 0x0c, 0x0a, 0x05, 0x04,
+ 0x00, 0x02, 0x00, 0x05, 0x12, 0x03, 0x24, 0x02, 0x07, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02,
+ 0x00, 0x01, 0x12, 0x03, 0x24, 0x08, 0x0c, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x00, 0x03,
+ 0x12, 0x03, 0x24, 0x0f, 0x10, 0x0a, 0xeb, 0x01, 0x0a, 0x04, 0x04, 0x00, 0x02, 0x01, 0x12, 0x03,
+ 0x29, 0x02, 0x15, 0x1a, 0xdd, 0x01, 0x20, 0x41, 0x20, 0x64, 0x65, 0x76, 0x65, 0x6c, 0x6f, 0x70,
+ 0x65, 0x72, 0x2d, 0x66, 0x61, 0x63, 0x69, 0x6e, 0x67, 0x20, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x20,
+ 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2c, 0x20, 0x77, 0x68, 0x69, 0x63, 0x68, 0x20, 0x73,
+ 0x68, 0x6f, 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x69, 0x6e, 0x20, 0x45, 0x6e, 0x67, 0x6c,
+ 0x69, 0x73, 0x68, 0x2e, 0x20, 0x41, 0x6e, 0x79, 0x0a, 0x20, 0x75, 0x73, 0x65, 0x72, 0x2d, 0x66,
+ 0x61, 0x63, 0x69, 0x6e, 0x67, 0x20, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x20, 0x6d, 0x65, 0x73, 0x73,
+ 0x61, 0x67, 0x65, 0x20, 0x73, 0x68, 0x6f, 0x75, 0x6c, 0x64, 0x20, 0x62, 0x65, 0x20, 0x6c, 0x6f,
+ 0x63, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x20, 0x61, 0x6e, 0x64, 0x20, 0x73, 0x65, 0x6e, 0x74,
+ 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65, 0x0a, 0x20, 0x5b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65,
+ 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x64, 0x65, 0x74, 0x61,
+ 0x69, 0x6c, 0x73, 0x5d, 0x5b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x72, 0x70, 0x63, 0x2e,
+ 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x5d, 0x20,
+ 0x66, 0x69, 0x65, 0x6c, 0x64, 0x2c, 0x20, 0x6f, 0x72, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x69,
+ 0x7a, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x74, 0x68, 0x65, 0x20, 0x63, 0x6c, 0x69, 0x65, 0x6e,
+ 0x74, 0x2e, 0x0a, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x05, 0x12, 0x03, 0x29, 0x02,
+ 0x08, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x01, 0x12, 0x03, 0x29, 0x09, 0x10, 0x0a,
+ 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x01, 0x03, 0x12, 0x03, 0x29, 0x13, 0x14, 0x0a, 0x79, 0x0a,
+ 0x04, 0x04, 0x00, 0x02, 0x02, 0x12, 0x03, 0x2d, 0x02, 0x2b, 0x1a, 0x6c, 0x20, 0x41, 0x20, 0x6c,
+ 0x69, 0x73, 0x74, 0x20, 0x6f, 0x66, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x20,
+ 0x74, 0x68, 0x61, 0x74, 0x20, 0x63, 0x61, 0x72, 0x72, 0x79, 0x20, 0x74, 0x68, 0x65, 0x20, 0x65,
+ 0x72, 0x72, 0x6f, 0x72, 0x20, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x20, 0x20, 0x54,
+ 0x68, 0x65, 0x72, 0x65, 0x20, 0x69, 0x73, 0x20, 0x61, 0x20, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e,
+ 0x20, 0x73, 0x65, 0x74, 0x20, 0x6f, 0x66, 0x0a, 0x20, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
+ 0x20, 0x74, 0x79, 0x70, 0x65, 0x73, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x41, 0x50, 0x49, 0x73, 0x20,
+ 0x74, 0x6f, 0x20, 0x75, 0x73, 0x65, 0x2e, 0x0a, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x02,
+ 0x04, 0x12, 0x03, 0x2d, 0x02, 0x0a, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x02, 0x06, 0x12,
+ 0x03, 0x2d, 0x0b, 0x1e, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x02, 0x01, 0x12, 0x03, 0x2d,
+ 0x1f, 0x26, 0x0a, 0x0c, 0x0a, 0x05, 0x04, 0x00, 0x02, 0x02, 0x03, 0x12, 0x03, 0x2d, 0x29, 0x2a,
+ 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+];
+// @@protoc_insertion_point(module)
\ No newline at end of file
diff --git a/flyteidl2/gen_utils/rust/src/lib.rs b/flyteidl2/gen_utils/rust/src/lib.rs
new file mode 100644
index 00000000000..de3f8f3d999
--- /dev/null
+++ b/flyteidl2/gen_utils/rust/src/lib.rs
@@ -0,0 +1,116 @@
+// mod test_foo;
+
+// mod serde;
+
+// mod serde_impl;
+
+use pyo3;
+use pyo3::prelude::*;
+// use crate::HeartbeatResponse;
+// include!("flyteidl.common.rs");
+// include!("flyteidl.workflow.rs");
+// include!("flyteidl.workflow.tonic.rs");
+// inculde!("flyteidl.logs.dataplane.rs");
+// include!("flyteidl.core.rs");
+// include!("google.rpc.rs");
+// include!("validate.rs");
+
+// use crate::*;
+// Re-export all generated protobuf modules
+pub mod flyteidl {
+
+ pub mod app {
+ include!("flyteidl2.app.rs");
+ }
+
+ pub mod auth {
+ include!("flyteidl2.auth.rs");
+ }
+
+ pub mod project {
+ include!("flyteidl2.project.rs");
+ }
+
+ pub mod common {
+ include!("flyteidl2.common.rs");
+ }
+
+ pub mod workflow {
+ include!("flyteidl2.workflow.rs");
+ }
+
+ pub mod logs {
+ pub mod dataplane {
+ include!("flyteidl2.logs.dataplane.rs");
+ }
+ }
+
+ pub mod core {
+ include!("flyteidl2.core.rs");
+ }
+
+ pub mod notification {
+ include!("flyteidl2.notification.rs");
+ }
+
+ pub mod task {
+ include!("flyteidl2.task.rs");
+ }
+
+ pub mod trigger {
+ include!("flyteidl2.trigger.rs");
+ }
+
+ pub mod secret {
+ include!("flyteidl2.secret.rs");
+ }
+}
+
+// use pyo3_prost::pyclass_for_prost_struct;
+pub mod google {
+ pub mod rpc {
+ include!("google.rpc.rs");
+ // pub mod serde {
+ // include!("google.rpc.serde.rs");
+ // }
+ }
+ pub mod protobuf {
+ include!(concat!(env!("OUT_DIR"), "/google.protobuf.rs"));
+ include!(concat!(env!("OUT_DIR"), "/google.protobuf.serde.rs"));
+ }
+}
+
+pub mod validate {
+ include!("validate.rs");
+ // pub mod serde {
+ // include!("validate.serde.rs");
+ // }
+}
+
+// Include the generated Box implementations
+include!(concat!(env!("OUT_DIR"), "/boxed_impls.rs"));
+// pub mod serde {
+// include!(concat!(env!("OUT_DIR"), "/serde_impls.rs"));
+// }
+pub mod pymodules {
+ include!(concat!(env!("OUT_DIR"), "/pymodules.rs"));
+}
+
+include!(concat!(env!("OUT_DIR"), "/serde_impls.rs"));
+// include!("serde_impl.rs");
+
+//
+// // Re-export commonly used types at the root level for convenience
+// pub use flyteidl::common::*;
+// pub use flyteidl::workflow::*;
+// pub use flyteidl::core::*;
+// pub use google::rpc::*;
+// pub use validate::*;
+// pub use crate::*;
+
+// #[pymodule]
+// fn pb_rust(_py: Python, m: &PyModule) -> PyResult<()> {
+// m.add_submodule(flyteidl::workflow::make_module(_py)?)?;
+// // ... all submodules ...
+// Ok(())
+// }
diff --git a/flyteidl2/gen_utils/rust/src/validate.rs b/flyteidl2/gen_utils/rust/src/validate.rs
new file mode 100644
index 00000000000..01f02e8ad87
--- /dev/null
+++ b/flyteidl2/gen_utils/rust/src/validate.rs
@@ -0,0 +1,3658 @@
+// @generated
+// This file is @generated by prost-build.
+/// FieldRules encapsulates the rules for each type of field. Depending on the
+/// field, the correct set should be used to ensure proper validations.
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct FieldRules {
+ #[prost(message, optional, tag="17")]
+ pub message: ::core::option::Option,
+ #[prost(oneof="field_rules::Type", tags="1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20, 21, 22")]
+ pub r#type: ::core::option::Option,
+}
+/// Nested message and enum types in `FieldRules`.
+pub mod field_rules {
+ #[pyo3::pyclass(dict, get_all, set_all)]
+ #[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Oneof)]
+ pub enum Type {
+ /// Scalar Field Types
+ #[prost(message, tag="1")]
+ Float(super::FloatRules),
+ #[prost(message, tag="2")]
+ Double(super::DoubleRules),
+ #[prost(message, tag="3")]
+ Int32(super::Int32Rules),
+ #[prost(message, tag="4")]
+ Int64(super::Int64Rules),
+ #[prost(message, tag="5")]
+ Uint32(super::UInt32Rules),
+ #[prost(message, tag="6")]
+ Uint64(super::UInt64Rules),
+ #[prost(message, tag="7")]
+ Sint32(super::SInt32Rules),
+ #[prost(message, tag="8")]
+ Sint64(super::SInt64Rules),
+ #[prost(message, tag="9")]
+ Fixed32(super::Fixed32Rules),
+ #[prost(message, tag="10")]
+ Fixed64(super::Fixed64Rules),
+ #[prost(message, tag="11")]
+ Sfixed32(super::SFixed32Rules),
+ #[prost(message, tag="12")]
+ Sfixed64(super::SFixed64Rules),
+ #[prost(message, tag="13")]
+ Bool(super::BoolRules),
+ #[prost(message, tag="14")]
+ String(super::StringRules),
+ #[prost(message, tag="15")]
+ Bytes(super::BytesRules),
+ /// Complex Field Types
+ #[prost(message, tag="16")]
+ Enum(super::EnumRules),
+ #[prost(message, tag="18")]
+ Repeated(::prost::alloc::boxed::Box),
+ #[prost(message, tag="19")]
+ Map(::prost::alloc::boxed::Box),
+ /// Well-Known Field Types
+ #[prost(message, tag="20")]
+ Any(super::AnyRules),
+ #[prost(message, tag="21")]
+ Duration(super::DurationRules),
+ #[prost(message, tag="22")]
+ Timestamp(super::TimestampRules),
+ }
+}
+/// FloatRules describes the constraints applied to `float` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct FloatRules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(float, optional, tag="1")]
+ pub r#const: ::core::option::Option,
+ /// Lt specifies that this field must be less than the specified value,
+ /// exclusive
+ #[prost(float, optional, tag="2")]
+ pub lt: ::core::option::Option,
+ /// Lte specifies that this field must be less than or equal to the
+ /// specified value, inclusive
+ #[prost(float, optional, tag="3")]
+ pub lte: ::core::option::Option,
+ /// Gt specifies that this field must be greater than the specified value,
+ /// exclusive. If the value of Gt is larger than a specified Lt or Lte, the
+ /// range is reversed.
+ #[prost(float, optional, tag="4")]
+ pub gt: ::core::option::Option,
+ /// Gte specifies that this field must be greater than or equal to the
+ /// specified value, inclusive. If the value of Gte is larger than a
+ /// specified Lt or Lte, the range is reversed.
+ #[prost(float, optional, tag="5")]
+ pub gte: ::core::option::Option,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(float, repeated, packed="false", tag="6")]
+ pub r#in: ::prost::alloc::vec::Vec,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(float, repeated, packed="false", tag="7")]
+ pub not_in: ::prost::alloc::vec::Vec,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="8")]
+ pub ignore_empty: ::core::option::Option,
+}
+/// DoubleRules describes the constraints applied to `double` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct DoubleRules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(double, optional, tag="1")]
+ pub r#const: ::core::option::Option,
+ /// Lt specifies that this field must be less than the specified value,
+ /// exclusive
+ #[prost(double, optional, tag="2")]
+ pub lt: ::core::option::Option,
+ /// Lte specifies that this field must be less than or equal to the
+ /// specified value, inclusive
+ #[prost(double, optional, tag="3")]
+ pub lte: ::core::option::Option,
+ /// Gt specifies that this field must be greater than the specified value,
+ /// exclusive. If the value of Gt is larger than a specified Lt or Lte, the
+ /// range is reversed.
+ #[prost(double, optional, tag="4")]
+ pub gt: ::core::option::Option,
+ /// Gte specifies that this field must be greater than or equal to the
+ /// specified value, inclusive. If the value of Gte is larger than a
+ /// specified Lt or Lte, the range is reversed.
+ #[prost(double, optional, tag="5")]
+ pub gte: ::core::option::Option,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(double, repeated, packed="false", tag="6")]
+ pub r#in: ::prost::alloc::vec::Vec,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(double, repeated, packed="false", tag="7")]
+ pub not_in: ::prost::alloc::vec::Vec,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="8")]
+ pub ignore_empty: ::core::option::Option,
+}
+/// Int32Rules describes the constraints applied to `int32` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct Int32Rules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(int32, optional, tag="1")]
+ pub r#const: ::core::option::Option,
+ /// Lt specifies that this field must be less than the specified value,
+ /// exclusive
+ #[prost(int32, optional, tag="2")]
+ pub lt: ::core::option::Option,
+ /// Lte specifies that this field must be less than or equal to the
+ /// specified value, inclusive
+ #[prost(int32, optional, tag="3")]
+ pub lte: ::core::option::Option,
+ /// Gt specifies that this field must be greater than the specified value,
+ /// exclusive. If the value of Gt is larger than a specified Lt or Lte, the
+ /// range is reversed.
+ #[prost(int32, optional, tag="4")]
+ pub gt: ::core::option::Option,
+ /// Gte specifies that this field must be greater than or equal to the
+ /// specified value, inclusive. If the value of Gte is larger than a
+ /// specified Lt or Lte, the range is reversed.
+ #[prost(int32, optional, tag="5")]
+ pub gte: ::core::option::Option,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(int32, repeated, packed="false", tag="6")]
+ pub r#in: ::prost::alloc::vec::Vec,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(int32, repeated, packed="false", tag="7")]
+ pub not_in: ::prost::alloc::vec::Vec,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="8")]
+ pub ignore_empty: ::core::option::Option,
+}
+/// Int64Rules describes the constraints applied to `int64` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct Int64Rules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(int64, optional, tag="1")]
+ pub r#const: ::core::option::Option,
+ /// Lt specifies that this field must be less than the specified value,
+ /// exclusive
+ #[prost(int64, optional, tag="2")]
+ pub lt: ::core::option::Option,
+ /// Lte specifies that this field must be less than or equal to the
+ /// specified value, inclusive
+ #[prost(int64, optional, tag="3")]
+ pub lte: ::core::option::Option,
+ /// Gt specifies that this field must be greater than the specified value,
+ /// exclusive. If the value of Gt is larger than a specified Lt or Lte, the
+ /// range is reversed.
+ #[prost(int64, optional, tag="4")]
+ pub gt: ::core::option::Option,
+ /// Gte specifies that this field must be greater than or equal to the
+ /// specified value, inclusive. If the value of Gte is larger than a
+ /// specified Lt or Lte, the range is reversed.
+ #[prost(int64, optional, tag="5")]
+ pub gte: ::core::option::Option,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(int64, repeated, packed="false", tag="6")]
+ pub r#in: ::prost::alloc::vec::Vec,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(int64, repeated, packed="false", tag="7")]
+ pub not_in: ::prost::alloc::vec::Vec,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="8")]
+ pub ignore_empty: ::core::option::Option,
+}
+/// UInt32Rules describes the constraints applied to `uint32` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct UInt32Rules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(uint32, optional, tag="1")]
+ pub r#const: ::core::option::Option,
+ /// Lt specifies that this field must be less than the specified value,
+ /// exclusive
+ #[prost(uint32, optional, tag="2")]
+ pub lt: ::core::option::Option,
+ /// Lte specifies that this field must be less than or equal to the
+ /// specified value, inclusive
+ #[prost(uint32, optional, tag="3")]
+ pub lte: ::core::option::Option,
+ /// Gt specifies that this field must be greater than the specified value,
+ /// exclusive. If the value of Gt is larger than a specified Lt or Lte, the
+ /// range is reversed.
+ #[prost(uint32, optional, tag="4")]
+ pub gt: ::core::option::Option,
+ /// Gte specifies that this field must be greater than or equal to the
+ /// specified value, inclusive. If the value of Gte is larger than a
+ /// specified Lt or Lte, the range is reversed.
+ #[prost(uint32, optional, tag="5")]
+ pub gte: ::core::option::Option,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(uint32, repeated, packed="false", tag="6")]
+ pub r#in: ::prost::alloc::vec::Vec,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(uint32, repeated, packed="false", tag="7")]
+ pub not_in: ::prost::alloc::vec::Vec,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="8")]
+ pub ignore_empty: ::core::option::Option,
+}
+/// UInt64Rules describes the constraints applied to `uint64` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct UInt64Rules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(uint64, optional, tag="1")]
+ pub r#const: ::core::option::Option,
+ /// Lt specifies that this field must be less than the specified value,
+ /// exclusive
+ #[prost(uint64, optional, tag="2")]
+ pub lt: ::core::option::Option,
+ /// Lte specifies that this field must be less than or equal to the
+ /// specified value, inclusive
+ #[prost(uint64, optional, tag="3")]
+ pub lte: ::core::option::Option,
+ /// Gt specifies that this field must be greater than the specified value,
+ /// exclusive. If the value of Gt is larger than a specified Lt or Lte, the
+ /// range is reversed.
+ #[prost(uint64, optional, tag="4")]
+ pub gt: ::core::option::Option,
+ /// Gte specifies that this field must be greater than or equal to the
+ /// specified value, inclusive. If the value of Gte is larger than a
+ /// specified Lt or Lte, the range is reversed.
+ #[prost(uint64, optional, tag="5")]
+ pub gte: ::core::option::Option,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(uint64, repeated, packed="false", tag="6")]
+ pub r#in: ::prost::alloc::vec::Vec,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(uint64, repeated, packed="false", tag="7")]
+ pub not_in: ::prost::alloc::vec::Vec,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="8")]
+ pub ignore_empty: ::core::option::Option,
+}
+/// SInt32Rules describes the constraints applied to `sint32` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct SInt32Rules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(sint32, optional, tag="1")]
+ pub r#const: ::core::option::Option,
+ /// Lt specifies that this field must be less than the specified value,
+ /// exclusive
+ #[prost(sint32, optional, tag="2")]
+ pub lt: ::core::option::Option,
+ /// Lte specifies that this field must be less than or equal to the
+ /// specified value, inclusive
+ #[prost(sint32, optional, tag="3")]
+ pub lte: ::core::option::Option,
+ /// Gt specifies that this field must be greater than the specified value,
+ /// exclusive. If the value of Gt is larger than a specified Lt or Lte, the
+ /// range is reversed.
+ #[prost(sint32, optional, tag="4")]
+ pub gt: ::core::option::Option,
+ /// Gte specifies that this field must be greater than or equal to the
+ /// specified value, inclusive. If the value of Gte is larger than a
+ /// specified Lt or Lte, the range is reversed.
+ #[prost(sint32, optional, tag="5")]
+ pub gte: ::core::option::Option,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(sint32, repeated, packed="false", tag="6")]
+ pub r#in: ::prost::alloc::vec::Vec,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(sint32, repeated, packed="false", tag="7")]
+ pub not_in: ::prost::alloc::vec::Vec,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="8")]
+ pub ignore_empty: ::core::option::Option,
+}
+/// SInt64Rules describes the constraints applied to `sint64` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct SInt64Rules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(sint64, optional, tag="1")]
+ pub r#const: ::core::option::Option,
+ /// Lt specifies that this field must be less than the specified value,
+ /// exclusive
+ #[prost(sint64, optional, tag="2")]
+ pub lt: ::core::option::Option,
+ /// Lte specifies that this field must be less than or equal to the
+ /// specified value, inclusive
+ #[prost(sint64, optional, tag="3")]
+ pub lte: ::core::option::Option,
+ /// Gt specifies that this field must be greater than the specified value,
+ /// exclusive. If the value of Gt is larger than a specified Lt or Lte, the
+ /// range is reversed.
+ #[prost(sint64, optional, tag="4")]
+ pub gt: ::core::option::Option,
+ /// Gte specifies that this field must be greater than or equal to the
+ /// specified value, inclusive. If the value of Gte is larger than a
+ /// specified Lt or Lte, the range is reversed.
+ #[prost(sint64, optional, tag="5")]
+ pub gte: ::core::option::Option,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(sint64, repeated, packed="false", tag="6")]
+ pub r#in: ::prost::alloc::vec::Vec,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(sint64, repeated, packed="false", tag="7")]
+ pub not_in: ::prost::alloc::vec::Vec,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="8")]
+ pub ignore_empty: ::core::option::Option,
+}
+/// Fixed32Rules describes the constraints applied to `fixed32` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct Fixed32Rules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(fixed32, optional, tag="1")]
+ pub r#const: ::core::option::Option,
+ /// Lt specifies that this field must be less than the specified value,
+ /// exclusive
+ #[prost(fixed32, optional, tag="2")]
+ pub lt: ::core::option::Option,
+ /// Lte specifies that this field must be less than or equal to the
+ /// specified value, inclusive
+ #[prost(fixed32, optional, tag="3")]
+ pub lte: ::core::option::Option,
+ /// Gt specifies that this field must be greater than the specified value,
+ /// exclusive. If the value of Gt is larger than a specified Lt or Lte, the
+ /// range is reversed.
+ #[prost(fixed32, optional, tag="4")]
+ pub gt: ::core::option::Option,
+ /// Gte specifies that this field must be greater than or equal to the
+ /// specified value, inclusive. If the value of Gte is larger than a
+ /// specified Lt or Lte, the range is reversed.
+ #[prost(fixed32, optional, tag="5")]
+ pub gte: ::core::option::Option,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(fixed32, repeated, packed="false", tag="6")]
+ pub r#in: ::prost::alloc::vec::Vec,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(fixed32, repeated, packed="false", tag="7")]
+ pub not_in: ::prost::alloc::vec::Vec,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="8")]
+ pub ignore_empty: ::core::option::Option,
+}
+/// Fixed64Rules describes the constraints applied to `fixed64` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct Fixed64Rules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(fixed64, optional, tag="1")]
+ pub r#const: ::core::option::Option,
+ /// Lt specifies that this field must be less than the specified value,
+ /// exclusive
+ #[prost(fixed64, optional, tag="2")]
+ pub lt: ::core::option::Option,
+ /// Lte specifies that this field must be less than or equal to the
+ /// specified value, inclusive
+ #[prost(fixed64, optional, tag="3")]
+ pub lte: ::core::option::Option,
+ /// Gt specifies that this field must be greater than the specified value,
+ /// exclusive. If the value of Gt is larger than a specified Lt or Lte, the
+ /// range is reversed.
+ #[prost(fixed64, optional, tag="4")]
+ pub gt: ::core::option::Option,
+ /// Gte specifies that this field must be greater than or equal to the
+ /// specified value, inclusive. If the value of Gte is larger than a
+ /// specified Lt or Lte, the range is reversed.
+ #[prost(fixed64, optional, tag="5")]
+ pub gte: ::core::option::Option,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(fixed64, repeated, packed="false", tag="6")]
+ pub r#in: ::prost::alloc::vec::Vec,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(fixed64, repeated, packed="false", tag="7")]
+ pub not_in: ::prost::alloc::vec::Vec,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="8")]
+ pub ignore_empty: ::core::option::Option,
+}
+/// SFixed32Rules describes the constraints applied to `sfixed32` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct SFixed32Rules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(sfixed32, optional, tag="1")]
+ pub r#const: ::core::option::Option,
+ /// Lt specifies that this field must be less than the specified value,
+ /// exclusive
+ #[prost(sfixed32, optional, tag="2")]
+ pub lt: ::core::option::Option,
+ /// Lte specifies that this field must be less than or equal to the
+ /// specified value, inclusive
+ #[prost(sfixed32, optional, tag="3")]
+ pub lte: ::core::option::Option,
+ /// Gt specifies that this field must be greater than the specified value,
+ /// exclusive. If the value of Gt is larger than a specified Lt or Lte, the
+ /// range is reversed.
+ #[prost(sfixed32, optional, tag="4")]
+ pub gt: ::core::option::Option,
+ /// Gte specifies that this field must be greater than or equal to the
+ /// specified value, inclusive. If the value of Gte is larger than a
+ /// specified Lt or Lte, the range is reversed.
+ #[prost(sfixed32, optional, tag="5")]
+ pub gte: ::core::option::Option,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(sfixed32, repeated, packed="false", tag="6")]
+ pub r#in: ::prost::alloc::vec::Vec,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(sfixed32, repeated, packed="false", tag="7")]
+ pub not_in: ::prost::alloc::vec::Vec,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="8")]
+ pub ignore_empty: ::core::option::Option,
+}
+/// SFixed64Rules describes the constraints applied to `sfixed64` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct SFixed64Rules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(sfixed64, optional, tag="1")]
+ pub r#const: ::core::option::Option,
+ /// Lt specifies that this field must be less than the specified value,
+ /// exclusive
+ #[prost(sfixed64, optional, tag="2")]
+ pub lt: ::core::option::Option,
+ /// Lte specifies that this field must be less than or equal to the
+ /// specified value, inclusive
+ #[prost(sfixed64, optional, tag="3")]
+ pub lte: ::core::option::Option,
+ /// Gt specifies that this field must be greater than the specified value,
+ /// exclusive. If the value of Gt is larger than a specified Lt or Lte, the
+ /// range is reversed.
+ #[prost(sfixed64, optional, tag="4")]
+ pub gt: ::core::option::Option,
+ /// Gte specifies that this field must be greater than or equal to the
+ /// specified value, inclusive. If the value of Gte is larger than a
+ /// specified Lt or Lte, the range is reversed.
+ #[prost(sfixed64, optional, tag="5")]
+ pub gte: ::core::option::Option,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(sfixed64, repeated, packed="false", tag="6")]
+ pub r#in: ::prost::alloc::vec::Vec,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(sfixed64, repeated, packed="false", tag="7")]
+ pub not_in: ::prost::alloc::vec::Vec,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="8")]
+ pub ignore_empty: ::core::option::Option,
+}
+/// BoolRules describes the constraints applied to `bool` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+pub struct BoolRules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(bool, optional, tag="1")]
+ pub r#const: ::core::option::Option,
+}
+/// StringRules describe the constraints applied to `string` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct StringRules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(string, optional, tag="1")]
+ pub r#const: ::core::option::Option<::prost::alloc::string::String>,
+ /// Len specifies that this field must be the specified number of
+ /// characters (Unicode code points). Note that the number of
+ /// characters may differ from the number of bytes in the string.
+ #[prost(uint64, optional, tag="19")]
+ pub len: ::core::option::Option,
+ /// MinLen specifies that this field must be the specified number of
+ /// characters (Unicode code points) at a minimum. Note that the number of
+ /// characters may differ from the number of bytes in the string.
+ #[prost(uint64, optional, tag="2")]
+ pub min_len: ::core::option::Option,
+ /// MaxLen specifies that this field must be the specified number of
+ /// characters (Unicode code points) at a maximum. Note that the number of
+ /// characters may differ from the number of bytes in the string.
+ #[prost(uint64, optional, tag="3")]
+ pub max_len: ::core::option::Option,
+ /// LenBytes specifies that this field must be the specified number of bytes
+ #[prost(uint64, optional, tag="20")]
+ pub len_bytes: ::core::option::Option,
+ /// MinBytes specifies that this field must be the specified number of bytes
+ /// at a minimum
+ #[prost(uint64, optional, tag="4")]
+ pub min_bytes: ::core::option::Option,
+ /// MaxBytes specifies that this field must be the specified number of bytes
+ /// at a maximum
+ #[prost(uint64, optional, tag="5")]
+ pub max_bytes: ::core::option::Option,
+ /// Pattern specifies that this field must match against the specified
+ /// regular expression (RE2 syntax). The included expression should elide
+ /// any delimiters.
+ #[prost(string, optional, tag="6")]
+ pub pattern: ::core::option::Option<::prost::alloc::string::String>,
+ /// Prefix specifies that this field must have the specified substring at
+ /// the beginning of the string.
+ #[prost(string, optional, tag="7")]
+ pub prefix: ::core::option::Option<::prost::alloc::string::String>,
+ /// Suffix specifies that this field must have the specified substring at
+ /// the end of the string.
+ #[prost(string, optional, tag="8")]
+ pub suffix: ::core::option::Option<::prost::alloc::string::String>,
+ /// Contains specifies that this field must have the specified substring
+ /// anywhere in the string.
+ #[prost(string, optional, tag="9")]
+ pub contains: ::core::option::Option<::prost::alloc::string::String>,
+ /// NotContains specifies that this field cannot have the specified substring
+ /// anywhere in the string.
+ #[prost(string, optional, tag="23")]
+ pub not_contains: ::core::option::Option<::prost::alloc::string::String>,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(string, repeated, tag="10")]
+ pub r#in: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(string, repeated, tag="11")]
+ pub not_in: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+ /// This applies to regexes HTTP_HEADER_NAME and HTTP_HEADER_VALUE to enable
+ /// strict header validation.
+ /// By default, this is true, and HTTP header validations are RFC-compliant.
+ /// Setting to false will enable a looser validations that only disallows
+ /// \r\n\0 characters, which can be used to bypass header matching rules.
+ #[prost(bool, optional, tag="25", default="true")]
+ pub strict: ::core::option::Option,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="26")]
+ pub ignore_empty: ::core::option::Option,
+ /// WellKnown rules provide advanced constraints against common string
+ /// patterns
+ #[prost(oneof="string_rules::WellKnown", tags="12, 13, 14, 15, 16, 17, 18, 21, 22, 24")]
+ pub well_known: ::core::option::Option,
+}
+/// Nested message and enum types in `StringRules`.
+pub mod string_rules {
+ /// WellKnown rules provide advanced constraints against common string
+ /// patterns
+ #[pyo3::pyclass(dict, get_all, set_all)]
+ #[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, Copy, PartialEq, ::prost::Oneof)]
+ pub enum WellKnown {
+ /// Email specifies that the field must be a valid email address as
+ /// defined by RFC 5322
+ #[prost(bool, tag="12")]
+ Email(bool),
+ /// Hostname specifies that the field must be a valid hostname as
+ /// defined by RFC 1034. This constraint does not support
+ /// internationalized domain names (IDNs).
+ #[prost(bool, tag="13")]
+ Hostname(bool),
+ /// Ip specifies that the field must be a valid IP (v4 or v6) address.
+ /// Valid IPv6 addresses should not include surrounding square brackets.
+ #[prost(bool, tag="14")]
+ Ip(bool),
+ /// Ipv4 specifies that the field must be a valid IPv4 address.
+ #[prost(bool, tag="15")]
+ Ipv4(bool),
+ /// Ipv6 specifies that the field must be a valid IPv6 address. Valid
+ /// IPv6 addresses should not include surrounding square brackets.
+ #[prost(bool, tag="16")]
+ Ipv6(bool),
+ /// Uri specifies that the field must be a valid, absolute URI as defined
+ /// by RFC 3986
+ #[prost(bool, tag="17")]
+ Uri(bool),
+ /// UriRef specifies that the field must be a valid URI as defined by RFC
+ /// 3986 and may be relative or absolute.
+ #[prost(bool, tag="18")]
+ UriRef(bool),
+ /// Address specifies that the field must be either a valid hostname as
+ /// defined by RFC 1034 (which does not support internationalized domain
+ /// names or IDNs), or it can be a valid IP (v4 or v6).
+ #[prost(bool, tag="21")]
+ Address(bool),
+ /// Uuid specifies that the field must be a valid UUID as defined by
+ /// RFC 4122
+ #[prost(bool, tag="22")]
+ Uuid(bool),
+ /// WellKnownRegex specifies a common well known pattern defined as a regex.
+ #[prost(enumeration="super::KnownRegex", tag="24")]
+ WellKnownRegex(i32),
+ }
+}
+/// BytesRules describe the constraints applied to `bytes` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct BytesRules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(bytes="vec", optional, tag="1")]
+ pub r#const: ::core::option::Option<::prost::alloc::vec::Vec>,
+ /// Len specifies that this field must be the specified number of bytes
+ #[prost(uint64, optional, tag="13")]
+ pub len: ::core::option::Option,
+ /// MinLen specifies that this field must be the specified number of bytes
+ /// at a minimum
+ #[prost(uint64, optional, tag="2")]
+ pub min_len: ::core::option::Option,
+ /// MaxLen specifies that this field must be the specified number of bytes
+ /// at a maximum
+ #[prost(uint64, optional, tag="3")]
+ pub max_len: ::core::option::Option,
+ /// Pattern specifies that this field must match against the specified
+ /// regular expression (RE2 syntax). The included expression should elide
+ /// any delimiters.
+ #[prost(string, optional, tag="4")]
+ pub pattern: ::core::option::Option<::prost::alloc::string::String>,
+ /// Prefix specifies that this field must have the specified bytes at the
+ /// beginning of the string.
+ #[prost(bytes="vec", optional, tag="5")]
+ pub prefix: ::core::option::Option<::prost::alloc::vec::Vec>,
+ /// Suffix specifies that this field must have the specified bytes at the
+ /// end of the string.
+ #[prost(bytes="vec", optional, tag="6")]
+ pub suffix: ::core::option::Option<::prost::alloc::vec::Vec>,
+ /// Contains specifies that this field must have the specified bytes
+ /// anywhere in the string.
+ #[prost(bytes="vec", optional, tag="7")]
+ pub contains: ::core::option::Option<::prost::alloc::vec::Vec>,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(bytes="vec", repeated, tag="8")]
+ pub r#in: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(bytes="vec", repeated, tag="9")]
+ pub not_in: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="14")]
+ pub ignore_empty: ::core::option::Option,
+ /// WellKnown rules provide advanced constraints against common byte
+ /// patterns
+ #[prost(oneof="bytes_rules::WellKnown", tags="10, 11, 12")]
+ pub well_known: ::core::option::Option,
+}
+/// Nested message and enum types in `BytesRules`.
+pub mod bytes_rules {
+ /// WellKnown rules provide advanced constraints against common byte
+ /// patterns
+ #[pyo3::pyclass(dict, get_all, set_all)]
+ #[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, Copy, PartialEq, ::prost::Oneof)]
+ pub enum WellKnown {
+ /// Ip specifies that the field must be a valid IP (v4 or v6) address in
+ /// byte format
+ #[prost(bool, tag="10")]
+ Ip(bool),
+ /// Ipv4 specifies that the field must be a valid IPv4 address in byte
+ /// format
+ #[prost(bool, tag="11")]
+ Ipv4(bool),
+ /// Ipv6 specifies that the field must be a valid IPv6 address in byte
+ /// format
+ #[prost(bool, tag="12")]
+ Ipv6(bool),
+ }
+}
+/// EnumRules describe the constraints applied to enum values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct EnumRules {
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(int32, optional, tag="1")]
+ pub r#const: ::core::option::Option,
+ /// DefinedOnly specifies that this field must be only one of the defined
+ /// values for this enum, failing on any undefined value.
+ #[prost(bool, optional, tag="2")]
+ pub defined_only: ::core::option::Option,
+ /// In specifies that this field must be equal to one of the specified
+ /// values
+ #[prost(int32, repeated, packed="false", tag="3")]
+ pub r#in: ::prost::alloc::vec::Vec,
+ /// NotIn specifies that this field cannot be equal to one of the specified
+ /// values
+ #[prost(int32, repeated, packed="false", tag="4")]
+ pub not_in: ::prost::alloc::vec::Vec,
+}
+/// MessageRules describe the constraints applied to embedded message values.
+/// For message-type fields, validation is performed recursively.
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, Copy, PartialEq, ::prost::Message)]
+pub struct MessageRules {
+ /// Skip specifies that the validation rules of this field should not be
+ /// evaluated
+ #[prost(bool, optional, tag="1")]
+ pub skip: ::core::option::Option,
+ /// Required specifies that this field must be set
+ #[prost(bool, optional, tag="2")]
+ pub required: ::core::option::Option,
+}
+/// RepeatedRules describe the constraints applied to `repeated` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct RepeatedRules {
+ /// MinItems specifies that this field must have the specified number of
+ /// items at a minimum
+ #[prost(uint64, optional, tag="1")]
+ pub min_items: ::core::option::Option,
+ /// MaxItems specifies that this field must have the specified number of
+ /// items at a maximum
+ #[prost(uint64, optional, tag="2")]
+ pub max_items: ::core::option::Option,
+ /// Unique specifies that all elements in this field must be unique. This
+ /// constraint is only applicable to scalar and enum types (messages are not
+ /// supported).
+ #[prost(bool, optional, tag="3")]
+ pub unique: ::core::option::Option,
+ /// Items specifies the constraints to be applied to each item in the field.
+ /// Repeated message fields will still execute validation against each item
+ /// unless skip is specified here.
+ #[prost(message, optional, boxed, tag="4")]
+ pub items: ::core::option::Option<::prost::alloc::boxed::Box>,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="5")]
+ pub ignore_empty: ::core::option::Option,
+}
+/// MapRules describe the constraints applied to `map` values
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct MapRules {
+ /// MinPairs specifies that this field must have the specified number of
+ /// KVs at a minimum
+ #[prost(uint64, optional, tag="1")]
+ pub min_pairs: ::core::option::Option,
+ /// MaxPairs specifies that this field must have the specified number of
+ /// KVs at a maximum
+ #[prost(uint64, optional, tag="2")]
+ pub max_pairs: ::core::option::Option,
+ /// NoSparse specifies values in this field cannot be unset. This only
+ /// applies to map's with message value types.
+ #[prost(bool, optional, tag="3")]
+ pub no_sparse: ::core::option::Option,
+ /// Keys specifies the constraints to be applied to each key in the field.
+ #[prost(message, optional, boxed, tag="4")]
+ pub keys: ::core::option::Option<::prost::alloc::boxed::Box>,
+ /// Values specifies the constraints to be applied to the value of each key
+ /// in the field. Message values will still have their validations evaluated
+ /// unless skip is specified here.
+ #[prost(message, optional, boxed, tag="5")]
+ pub values: ::core::option::Option<::prost::alloc::boxed::Box>,
+ /// IgnoreEmpty specifies that the validation rules of this field should be
+ /// evaluated only if the field is not empty
+ #[prost(bool, optional, tag="6")]
+ pub ignore_empty: ::core::option::Option,
+}
+/// AnyRules describe constraints applied exclusively to the
+/// `google.protobuf.Any` well-known type
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct AnyRules {
+ /// Required specifies that this field must be set
+ #[prost(bool, optional, tag="1")]
+ pub required: ::core::option::Option,
+ /// In specifies that this field's `type_url` must be equal to one of the
+ /// specified values.
+ #[prost(string, repeated, tag="2")]
+ pub r#in: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+ /// NotIn specifies that this field's `type_url` must not be equal to any of
+ /// the specified values.
+ #[prost(string, repeated, tag="3")]
+ pub not_in: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+}
+/// DurationRules describe the constraints applied exclusively to the
+/// `google.protobuf.Duration` well-known type
+#[pyo3::pyclass(dict, get_all, set_all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct DurationRules {
+ /// Required specifies that this field must be set
+ #[prost(bool, optional, tag="1")]
+ pub required: ::core::option::Option,
+ /// Const specifies that this field must be exactly the specified value
+ #[prost(message, optional, tag="2")]
+ pub r#const: ::core::option::Option,
+ /// Lt specifies that this field must be less than the specified value,
+ /// exclusive
+ #[prost(message, optional, tag="3")]
+ pub lt: ::core::option::Option,
+ /// Lt specifies that this field must be less than the specified value,
+ /// inclusive
+ #[prost(message, optional, tag="4")]
+ pub lte: ::core::option::Option,
+ /// Gt specifies that this field must be greater than the specified value,
+ /// exclusive
+ #[prost(message, optional, tag="5")]
+ pub gt: ::core::option::Option,
+ /// Gte specifies that this field must be greater than the specified value,
+ /// inclusive
+ #[prost(message, optional, tag="6")]
+ pub gte: ::core::option::Option