WIP: gui, mpc + update dataflow single pod for manifest

Author: ilyario

.github/workflows/mcp-dataflow.yml

@@ -0,0 +1,220 @@
+name: Build and Push MCP DataFlow Server
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'mcp-dataflow/**'
+      - '.github/workflows/mcp-dataflow.yml'
+    tags:
+      - 'mcp-v*'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'mcp-dataflow/**'
+      - '.github/workflows/mcp-dataflow.yml'
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ilyario/mcp-dataflow
+  GO_VERSION: '1.21'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Install Task
+        uses: arduino/setup-task@v1
+        with:
+          version: 3.x
+
+      - name: Download dependencies
+        working-directory: ./mcp-dataflow
+        run: task deps
+
+      - name: Run tests
+        working-directory: ./mcp-dataflow
+        run: task test
+
+      - name: Run linter
+        working-directory: ./mcp-dataflow
+        run: task lint || true
+
+  build-binaries:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, darwin, windows]
+        goarch: [amd64, arm64]
+        exclude:
+          - goos: windows
+            goarch: arm64
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Install Task
+        uses: arduino/setup-task@v1
+        with:
+          version: 3.x
+
+      - name: Download dependencies
+        working-directory: ./mcp-dataflow
+        run: task deps
+
+      - name: Build binary
+        working-directory: ./mcp-dataflow
+        env:
+          GOOS: ${{ matrix.goos }}
+          GOARCH: ${{ matrix.goarch }}
+        run: |
+          OUTPUT="dist/mcp-dataflow-${GOOS}-${GOARCH}"
+          if [ "$GOOS" = "windows" ]; then
+            OUTPUT="${OUTPUT}.exe"
+          fi
+          CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} go build -a -installsuffix cgo -o ${OUTPUT} cmd/server/main.go
+
+      - name: Upload binaries
+        uses: actions/upload-artifact@v4
+        with:
+          name: binaries-${{ matrix.goos }}-${{ matrix.goarch }}
+          path: mcp-dataflow/dist/mcp-dataflow-${{ matrix.goos }}-${{ matrix.goarch }}*
+
+  build-and-push:
+    runs-on: ubuntu-latest
+    needs: [test]
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha,prefix={{branch}}-
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=raw,value=mcp-latest,enable={{is_default_branch}}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./mcp-dataflow
+          file: ./mcp-dataflow/Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  create-release:
+    needs: [build-binaries, build-and-push]
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/mcp-v')
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Get tag name
+        id: tag
+        run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT
+
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+
+      - name: Prepare release assets
+        run: |
+          mkdir -p release-assets
+          find dist -type f -executable -o -name "*.exe" | while read file; do
+            cp "$file" release-assets/
+          done
+
+      - name: Generate release notes
+        id: release_notes
+        run: |
+          PREVIOUS_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "")
+          if [ -z "$PREVIOUS_TAG" ]; then
+            NOTES=$(git log --pretty=format:"- %s (%h)" --no-merges)
+          else
+            NOTES=$(git log ${PREVIOUS_TAG}..HEAD --pretty=format:"- %s (%h)" --no-merges)
+          fi
+          echo "notes<<EOF" >> $GITHUB_OUTPUT
+          echo "$NOTES" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+      - name: Create Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ steps.tag.outputs.TAG_NAME }}
+          name: MCP DataFlow Server ${{ steps.tag.outputs.TAG_NAME }}
+          body: |
+            ## Что нового в ${{ steps.tag.outputs.TAG_NAME }}
+
+            ${{ steps.release_notes.outputs.notes }}
+
+            ## Docker образ
+
+            Docker образ доступен по адресу:
+            ```
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG_NAME }}
+            ```
+
+            ## Установка
+
+            ### Docker
+            ```bash
+            docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG_NAME }}
+            ```
+
+            ### Бинарник
+
+            Скачайте бинарник для вашей платформы из раздела Assets ниже.
+          files: release-assets/*
+          draft: false
+          prerelease: false
+          generate_release_notes: true

Dockerfile

@@ -10,18 +10,21 @@ RUN apk add --no-cache git make
 COPY go.mod go.mod
 COPY go.sum go.sum
 
-# Copy nessie-client directory (required for replace directive)
-COPY pkg/nessie-client/go.mod pkg/nessie-client/go.mod
-
 # Download dependencies
 RUN go mod download
 
 # Copy source code
 COPY . .
 
-# Build
+# Build manager (operator)
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
 
+# Build processor
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o processor cmd/processor/main.go
+
+# Build GUI server
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o gui-server cmd/gui-server/main.go
+
 # Final stage
 FROM alpine:latest
 
@@ -30,6 +33,8 @@ WORKDIR /
 RUN apk --no-cache add ca-certificates
 
 COPY --from=builder /workspace/manager .
+COPY --from=builder /workspace/processor .
+COPY --from=builder /workspace/gui-server .
 
 ENTRYPOINT ["/manager"]
 

README.md

@@ -16,6 +16,9 @@ Kubernetes operator for streaming data between different sources (Kafka, Postgre
   - SnakeCase - convert field names to snake_case
   - CamelCase - convert field names to CamelCase
 - **Kubernetes Secrets Support**: Configure connectors using `SecretRef` for secure credential management
+- **Per-Resource Pod Deployment**: Each DataFlow resource creates a separate pod (Deployment) for processing
+- **Resource Management**: Configure CPU and memory resources for processor pods
+- **Pod Placement Control**: Configure nodeSelector, affinity, and tolerations for fine-grained pod placement
 
 ## Quick Start
 
@@ -189,6 +192,35 @@ make test
 make test-integration
 ```
 
+## Web GUI
+
+DataFlow включает веб-интерфейс для управления манифестами, просмотра логов и мониторинга метрик.
+
+### Запуск GUI сервера
+
+```bash
+go run cmd/gui-server/main.go --bind-address :8080
+```
+
+Или с параметрами:
+
+```bash
+go run cmd/gui-server/main.go \
+  --bind-address :8080 \
+  --kubeconfig ~/.kube/config \
+  --log-level info
+```
+
+### Использование
+
+1. Откройте браузер и перейдите на `http://localhost:8080`
+2. Используйте вкладки для:
+   - **Манифесты**: Управление DataFlow ресурсами (создание, просмотр, обновление, удаление)
+   - **Логи**: Просмотр логов обработки в реальном времени
+   - **Метрики**: Мониторинг статуса обработки, количества обработанных сообщений и ошибок
+
+Подробнее см. [cmd/gui-server/README.md](cmd/gui-server/README.md)
+
 ## Security
 
 DataFlow Operator supports configuring connectors from Kubernetes Secrets through `SecretRef` fields. This allows:

api/v1/dataflow_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -31,6 +32,26 @@ type DataFlowSpec struct {
 	// Transformations is a list of transformations to apply to messages
 	// +optional
 	Transformations []TransformationSpec `json:"transformations,omitempty"`
+
+	// Errors defines the error sink for messages that failed to be written to the main sink
+	// +optional
+	Errors *SinkSpec `json:"errors,omitempty"`
+
+	// Resources defines the resource requirements for the processor pod
+	// +optional
+	Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
+
+	// NodeSelector is a selector which must be true for the pod to fit on a node
+	// +optional
+	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
+
+	// Affinity is a group of affinity scheduling rules
+	// +optional
+	Affinity *corev1.Affinity `json:"affinity,omitempty"`
+
+	// Tolerations are attached to tolerate any taint that matches the triple <key,value,effect>
+	// +optional
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
 }
 
 // SourceSpec defines the source configuration