feat: Large File Support — proxy, operator, SDKs, and demos

.claude/commands/review.md

@@ -0,0 +1,48 @@
+---
+description: Performs thorough code review of current changes, looking for regressions, subtle bugs, and design issues like a staff engineer.
+mode: primary
+temperature: 0.1
+tools:
+  write: false
+  edit: false
+  bash: true
+permission:
+  bash:
+    "*": deny
+    "grep *": allow
+    "go test*": allow
+    "go build*": allow
+    "go vet*": allow
+    "go fmt*": allow
+    "git *": allow
+    "git push *": deny
+    "git reset *": deny
+---
+
+You are a staff-level engineer performing a thorough review of the current changes in this repository.
+
+Your job is to identify problems and simplification opportunities, not to make changes. Read the diff carefully, then explore the surrounding code to understand the full context before forming opinions. Before diving into details, step back and question whether the approach itself makes sense — does it actually achieve its intended goal, and is there a fundamentally better way to solve the same problem? Assume tests and the build already pass. Flag complexity issues in surrounding code when the change interacts with it, but do not suggest improvements completely unrelated to the changes under review.
+
+Focus on:
+
+- **Regressions**: Does this change break existing behavior? Look at callers, tests, and interfaces that depend on modified code.
+- **Concurrency issues**: Race conditions, missing locks, unsafe shared state, goroutine leaks.
+- **Error handling**: Swallowed errors, missing error propagation, inconsistent error behavior compared to neighboring code.
+- **Edge cases**: Nil pointers, empty slices, integer overflow, off-by-one errors, zero-value traps.
+- **Contract violations**: Does the change respect the implicit contracts of the interfaces and functions it touches? Are invariants preserved?
+- **Resource leaks**: Unclosed connections, files, or channels. Missing deferred cleanup.
+- **Behavioral inconsistencies**: Does the new code behave differently from similar patterns already in the codebase?
+- **Architecture and complexity**: Does the change introduce or reveal tight coupling, layering violations, misplaced responsibilities, unnecessary indirection, redundant abstractions, or duplicated logic? Would the change be significantly simpler under a different structural arrangement? Could touched code paths be expressed more directly?
+- **Test value**: Are added tests low-value (testing trivial behavior, duplicating existing coverage, or tightly coupled to implementation details)? Are there overlapping tests that could be consolidated? Are high-value tests missing — particularly for edge cases, error paths, concurrency, and integration boundaries that the change affects?
+
+Present your findings in two sections:
+
+## Issues
+
+Numbered list sorted by impact. For each: location (file:line), what is wrong and how it manifests, severity (critical/high/medium/low), and a concrete recommendation.
+
+## Simplification Opportunities
+
+Numbered list sorted by impact. For each: what is unnecessarily complex, where, what a simpler version looks like, and what improves as a result.
+
+If either section has no items, say so explicitly. Do not invent problems or fabricate opportunities.

.github/workflows/ci.yml

@@ -57,6 +57,28 @@ jobs:
       - name: Run go test -race ./...
         run: go test -race ./...
 
+  build-lfs-proxy:
+    name: Build LFS Proxy
+    runs-on: ubuntu-latest
+    env:
+      GOCACHE: ${{ github.workspace }}/.gocache
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4
+
+      - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+
+      - name: Prepare Go build cache
+        run: mkdir -p "$GOCACHE"
+
+      - name: Build lfs-proxy
+        run: go build -o lfs-proxy ./cmd/lfs-proxy
+
+      - name: Run lfs-proxy tests
+        run: go test ./cmd/lfs-proxy/...
+
   go-coverage:
     name: Go Coverage Gate
     runs-on: ubuntu-latest
@@ -76,6 +98,27 @@ jobs:
       - name: Enforce coverage floor
         run: bash hack/check_coverage.sh 45
 
+
+  e2e-lfs-proxy:
+    name: LFS Proxy E2E
+    runs-on: ubuntu-latest
+    env:
+      GOCACHE: ${{ github.workspace }}/.gocache
+      KAFSCALE_E2E: "1"
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4
+
+      - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+
+      - name: Prepare Go build cache
+        run: mkdir -p "$GOCACHE"
+
+      - name: Run LFS proxy E2E tests
+        run: go test -tags=e2e ./test/e2e -run LfsProxy
+
   helm-lint:
     name: Helm Lint
     runs-on: ubuntu-latest

.github/workflows/codeql.yml

@@ -51,6 +51,12 @@ jobs:
         with:
           languages: ${{ matrix.language }}
           queries: security-extended,security-and-quality
+          config: |
+            paths-ignore:
+              - '**/node_modules/**'
+              - '**/target/**'
+              - '**/*.egg-info/**'
+              - 'third_party/**'
 
       - name: Autobuild
         uses: github/codeql-action/autobuild@v4

.github/workflows/docker.yml

@@ -36,6 +36,10 @@ jobs:
             image: kafscale-broker
             context: .
             file: deploy/docker/broker.Dockerfile
+          - name: lfs-proxy
+            image: kafscale-lfs-proxy
+            context: .
+            file: deploy/docker/lfs-proxy.Dockerfile
           - name: operator
             image: kafscale-operator
             context: .

.github/workflows/stage-release.yml

@@ -0,0 +1,83 @@
+# Copyright 2026 Alexander Alten (novatechflow), NovaTechflow (novatechflow.com).
+# This project is supported and financed by Scalytics, Inc. (www.scalytics.io).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Stage Release (Local Registry)
+
+on:
+  workflow_dispatch:
+    inputs:
+      registry:
+        description: "Target registry host:port (e.g., 192.168.0.131:5000)"
+        required: true
+        default: "192.168.0.131:5000"
+      tag:
+        description: "Image tag to publish (e.g., stage-2026-02-04)"
+        required: true
+        default: "stage"
+
+permissions: read-all
+
+jobs:
+  stage-release:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - name: broker
+            image: kafscale-broker
+            context: .
+            file: deploy/docker/broker.Dockerfile
+          - name: lfs-proxy
+            image: kafscale-lfs-proxy
+            context: .
+            file: deploy/docker/lfs-proxy.Dockerfile
+          - name: operator
+            image: kafscale-operator
+            context: .
+            file: deploy/docker/operator.Dockerfile
+          - name: console
+            image: kafscale-console
+            context: .
+            file: deploy/docker/console.Dockerfile
+          - name: etcd-tools
+            image: kafscale-etcd-tools
+            context: .
+            file: deploy/docker/etcd-tools.Dockerfile
+          - name: iceberg-processor
+            image: kafscale-iceberg-processor
+            context: addons/processors/iceberg-processor
+            file: addons/processors/iceberg-processor/Dockerfile
+          - name: sql-processor
+            image: kafscale-sql-processor
+            context: addons/processors/sql-processor
+            file: addons/processors/sql-processor/Dockerfile
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+
+      - name: Build and push (local registry)
+        uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5
+        with:
+          context: ${{ matrix.context }}
+          file: ${{ matrix.file }}
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ inputs.registry }}/kafscale/${{ matrix.image }}:${{ inputs.tag }}

.gitignore

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+Makefile-MK # Ignore build and IDE files
+
 # Binaries
 /bin/
 /dist/
@@ -31,6 +33,9 @@
 *.key
 coverage*.out
 .build/
+target/
+**/target/
+spark-warehouse/
 
 # Local Go cache (use GOCACHE=.gocache for hermetic builds/tests)
 .gocache/
@@ -79,3 +84,27 @@ proto/**/*.swagger.json
 _site/
 Gemfile
 Gemfile.lock
+
+# Ignore demo node_modules
+examples/E50_JS-kafscale-demo/node_modules/
+
+# Go compiled binaries (top-level)
+/e2e-client
+/lfs-proxy
+/proxy
+
+# Java build artifacts
+target/
+dependency-reduced-pom.xml
+
+# JavaScript/Node.js
+node_modules/
+package-lock.json
+
+# Python
+__pycache__/
+*.pyc
+*.egg-info/
+
+# Test output
+records.txt