diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index f6a7fa6..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: CI - -on: - push: - branches: [main, develop] - pull_request: - branches: [main, develop] - -jobs: - test: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - version: "latest" - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - # Create virtual environment - uv venv - source .venv/bin/activate - # Install dependencies without editable package (workaround for hatchling issue) - uv pip install nats-py aiohttp - uv pip install pytest pytest-asyncio black ruff mypy - # Set PYTHONPATH for imports - echo "PYTHONPATH=src" >> $GITHUB_ENV - echo "VIRTUAL_ENV=$PWD/.venv" >> $GITHUB_ENV - - - name: Start NATS with JetStream - run: | - docker run -d --name nats-js \ - -p 4222:4222 \ - -p 8222:8222 \ - nats:latest -js -m 8222 - # Wait for NATS to be ready - for i in {1..30}; do - if timeout 1 bash -c "cat < /dev/null > /dev/tcp/localhost/4222" 2>/dev/null; then - echo "NATS is ready" - exit 0 - fi - echo "Waiting for NATS... ($i/30)" - sleep 1 - done - echo "NATS failed to start" - exit 1 - - - name: Cleanup NATS - if: always() - run: docker rm -f nats-js || true - - - name: Run tests - run: | - source .venv/bin/activate - PYTHONPATH=src pytest tests/ -v - env: - NATS_URL: nats://localhost:4222 - STREAM_NAME: droq-stream - - - name: Check formatting - run: | - source .venv/bin/activate - black --check src/ tests/ - - - name: Lint - run: | - source .venv/bin/activate - ruff check src/ tests/ - - docker: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build Docker image - run: docker build -t droq-node-template:test . - diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml new file mode 100644 index 0000000..9e0c436 --- /dev/null +++ b/.github/workflows/docker-ci.yml @@ -0,0 +1,239 @@ +name: Docker Build and Publish + +on: + push: + branches: + - main + - develop + tags: + - 'v*' + pull_request: + branches: + - main + - develop + +env: + # Set your organization name here + REGISTRY: ghcr.io + IMAGE_NAME: droq-math-executor-node + +jobs: + # Test build on pull requests (no push) + test-build: + name: Test Docker Build + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Test build Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + push: false + load: true + tags: ${{ env.IMAGE_NAME }}:test + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Test container startup + run: | + docker run -d --name test-container -p 8003:8003 ${{ env.IMAGE_NAME }}:test + sleep 10 + + # Test health endpoint + if ! curl -f http://localhost:8003/health; then + echo "Health check failed" + docker logs test-container + exit 1 + fi + + # Test API endpoint + response=$(curl -s -X POST http://localhost:8003/api/v1/execute \ + -H "Content-Type: application/json" \ + -d '{ + "component_state": { + "component_class": "DFXMultiplyComponent", + "component_module": "dfx.math.component.multiply", + "parameters": { "number1": 5.0, "number2": 3.0 } + }, + "method_name": "multiply", + "is_async": false + }') + + if ! echo "$response" | grep -q '"result":15.0'; then + echo "API test failed" + echo "Response: $response" + docker logs test-container + exit 1 + fi + + docker stop test-container + docker rm test-container + + # Build and publish on push to main/develop or tags + build-and-publish: + name: Build and Publish Docker Image + runs-on: ubuntu-latest + if: github.event_name == 'push' + permissions: + contents: read + packages: write + actions: read + id-token: write + outputs: + image: ${{ steps.meta.outputs.tags }} + digest: ${{ steps.build.outputs.digest }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }} + tags: | + # Latest tag for main branch + type=ref,event=branch,enable={{is_default_branch}},suffix=-latest + # Branch tags for develop + type=ref,event=branch,enable={{is_default_branch}},suffix=main + type=ref,event=branch,limit=1,suffix=develop + # Version tags for releases + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + # Commit SHA for unique identification + type=sha,prefix={{branch}}- + # Pull request tags + type=ref,event=pr,suffix=pr + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push Docker image + id: build + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 + # Ensure private registry configuration + provenance: true + sbom: true + + - name: Generate SBOM + uses: anchore/sbom-action@v0 + with: + image: ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }} + format: spdx-json + output-file: sbom.spdx.json + + - name: Upload SBOM as artifact + uses: actions/upload-artifact@v4 + with: + name: sbom-${{ github.sha }} + path: sbom.spdx.json + retention-days: 30 + + - name: Verify image in private registry + run: | + echo "Verifying image is accessible in private registry..." + # Test that we can pull the image we just pushed + docker pull ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}@${{ steps.build.outputs.digest }} + echo "✅ Image successfully verified in private registry" + + # List all tags for this image + echo "Image tags pushed:" + echo "${{ steps.meta.outputs.tags }}" | tr ',' '\n' | while read tag; do + echo " - $tag" + done + + # Create GitHub release on tag push + create-release: + name: Create GitHub Release + runs-on: ubuntu-latest + needs: build-and-publish + if: startsWith(github.ref, 'refs/tags/v') + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Extract release notes + id: release-notes + run: | + # Get tag without 'v' prefix + TAG="${GITHUB_REF#refs/tags/v}" + echo "version=$TAG" >> $GITHUB_OUTPUT + + # Create release notes from git commits since last tag + PREVIOUS_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "") + if [ -n "$PREVIOUS_TAG" ]; then + echo "changelog<> $GITHUB_OUTPUT + git log --pretty=format:"- %s (%h)" $PREVIOUS_TAG..HEAD >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + else + echo "changelog<> $GITHUB_OUTPUT + echo "Initial release" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + fi + + - name: Create Release + uses: softprops/action-gh-release@v2 + with: + name: Release ${{ steps.release-notes.outputs.version }} + body: | + ## Droq Math Executor Node v${{ steps.release-notes.outputs.version }} + + ### Docker Images + - `ghcr.io/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ steps.release-notes.outputs.version }}` + - `ghcr.io/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:latest` + + ### Installation + ```bash + docker pull ghcr.io/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ steps.release-notes.outputs.version }} + ``` + + ### Changelog + ${{ steps.release-notes.outputs.changelog }} + + ### Docker Compose + ```yaml + services: + math-executor: + image: ghcr.io/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ steps.release-notes.outputs.version }} + environment: + - HOST=0.0.0.0 + - PORT=8003 + - LOG_LEVEL=INFO + ports: + - "8003:8003" + restart: unless-stopped + ``` + generate_release_notes: true + draft: false + prerelease: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + \ No newline at end of file diff --git a/.github/workflows/scan-image.yml b/.github/workflows/scan-image.yml new file mode 100644 index 0000000..2ff9eea --- /dev/null +++ b/.github/workflows/scan-image.yml @@ -0,0 +1,101 @@ +name: Container Image Security Scan + +on: + schedule: + # Run daily at 2 AM UTC + - cron: '0 2 * * *' + workflow_dispatch: + inputs: + image_tag: + description: 'Image tag to scan (defaults to latest)' + required: false + default: 'latest' + image_ref: + description: 'Specific image reference (digest) to scan' + required: false + default: '' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: droq-math-executor-node + +jobs: + scan: + name: Security Scan + runs-on: ubuntu-latest + if: github.event.repository.is_public == true || github.event_name == 'workflow_dispatch' + permissions: + contents: read + security-events: write + packages: read + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Determine image to scan + id: image + run: | + if [[ -n "${{ github.event.inputs.image_ref }}" ]]; then + echo "image=${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}@${{ github.event.inputs.image_ref }}" >> $GITHUB_OUTPUT + else + TAG="${{ github.event.inputs.image_tag || 'latest' }}" + echo "image=${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${TAG}" >> $GITHUB_OUTPUT + fi + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ${{ steps.image.outputs.image }} + format: 'sarif' + output: 'trivy-results.sarif' + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + if: always() + with: + sarif_file: 'trivy-results.sarif' + category: container-scan + + - name: Generate security report + if: always() + run: | + echo "## Security Scan Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Image:** ${{ steps.image.outputs.image }}" >> $GITHUB_STEP_SUMMARY + echo "**Date:** $(date -u)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ -f "trivy-results.sarif" ]; then + # Count vulnerabilities by severity + CRITICAL=$(cat trivy-results.sarif | jq '.results[].runs[0].results | map(select(.level == "error")) | length' 2>/dev/null || echo "0") + HIGH=$(cat trivy-results.sarif | jq '.results[].runs[0].results | map(select(.level == "warning")) | length' 2>/dev/null || echo "0") + MEDIUM=$(cat trivy-results.sarif | jq '.results[].runs[0].results | map(select(.level == "note")) | length' 2>/dev/null || echo "0") + LOW=$(cat trivy-results.sarif | jq '.results[].runs[0].results | map(select(.level == "info")) | length' 2>/dev/null || echo "0") + + echo "| Severity | Count |" >> $GITHUB_STEP_SUMMARY + echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| 🔴 Critical | $CRITICAL |" >> $GITHUB_STEP_SUMMARY + echo "| 🟠 High | $HIGH |" >> $GITHUB_STEP_SUMMARY + echo "| 🟡 Medium | $MEDIUM |" >> $GITHUB_STEP_SUMMARY + echo "| 🔵 Low | $LOW |" >> $GITHUB_STEP_SUMMARY + else + echo "❌ Scan failed or no results generated" >> $GITHUB_STEP_SUMMARY + fi + + dependency-review: + name: Dependency Review + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + permissions: + contents: read + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Dependency Review + uses: actions/dependency-review-action@v4 + with: + fail-on-severity: moderate \ No newline at end of file diff --git a/DOCKERHUB_README.md b/DOCKERHUB_README.md new file mode 100644 index 0000000..5550a2c --- /dev/null +++ b/DOCKERHUB_README.md @@ -0,0 +1,96 @@ +# Droq Math Executor Node + +A simple executor node for basic math operations in the Droq workflow engine, using the **dfx (Droqflow Executor)** framework. + +## Overview + +This Docker image provides a FastAPI-based service that can execute math components within the Droq workflow ecosystem. It's built using the dfx framework and supports multiplication operations via the `DFXMultiplyComponent`. + +## Quick Start + +```bash +# Pull the image +docker pull ghcr.io/droq/dfx-math-executor-node:latest + +# Run with docker-compose (recommended) +curl -O https://raw.githubusercontent.com/droq/dfx-math-executor-node/main/compose.yml +curl -O https://raw.githubusercontent.com/droq/dfx-math-executor-node/main/.env +docker compose up -d + +# Or run with docker directly +docker run -d \ + --name droq-math-executor-node \ + -p 8003:8003 \ + -e HOST=0.0.0.0 \ + -e PORT=8003 \ + -e LOG_LEVEL=INFO \ + ghcr.io/droq/dfx-math-executor-node:latest +``` + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `HOST` | `0.0.0.0` | Host to bind the service to | +| `PORT` | `8003` | Port inside the container | +| `LOG_LEVEL` | `INFO` | Logging level (DEBUG, INFO, WARNING, ERROR) | + +## API Endpoints + +- `GET /health` - Health check +- `GET /` - Service information +- `POST /api/v1/execute` - Execute component methods + +## Example Usage + +```bash +# Test health check +curl http://localhost:8003/health + +# Execute multiplication (5 × 3 = 15) +curl -X POST http://localhost:8003/api/v1/execute \ + -H "Content-Type: application/json" \ + -d '{ + "component_state": { + "component_class": "DFXMultiplyComponent", + "component_module": "dfx.math.component.multiply", + "parameters": { + "number1": 5.0, + "number2": 3.0 + } + }, + "method_name": "multiply", + "is_async": false + }' +``` + +## Tags + +- `latest` - Latest stable release from main branch +- `main` - Latest build from main branch +- `develop` - Latest build from develop branch +- `vX.Y.Z` - Specific version releases +- `main-` - Specific commits from main branch + +## Architecture + +- **Base Image**: `python:3.11-slim` +- **Framework**: FastAPI with uvicorn +- **Dependencies**: Pydantic, httpx, nats-py, python-dotenv +- **Multi-platform**: Supports `linux/amd64` and `linux/arm64` + +## Security + +- Uses non-root user `app` inside container +- Minimal dependencies and base image +- Regular security updates via automated builds + +## Support + +For issues and questions: +- Repository: https://github.com/droq/dfx-math-executor-node +- Documentation: https://github.com/droq/dfx-math-executor-node#readme + +## License + +See the [LICENSE](https://github.com/droq/dfx-math-executor-node/blob/main/LICENSE) file for details. \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f102faa --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM python:3.11-slim + +WORKDIR /app + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Copy application files +COPY pyproject.toml README.md* ./ +COPY src/ ./src/ +COPY dfx/ ./dfx/ + +# Install dependencies directly +RUN pip install fastapi uvicorn pydantic httpx nats-py python-dotenv + +# Skip the problematic pip install -e . + +# Create non-root user +RUN useradd --create-home --shell /bin/bash app && \ + chown -R app:app /app +USER app + +# Expose port +EXPOSE 8003 + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD curl -f http://localhost:8003/health || exit 1 + +# Run the application directly +CMD ["python", "-c", "import sys; sys.path.append('/app/src'); sys.argv = ['main.py', '8003']; from math_executor.main import main; main()"] \ No newline at end of file diff --git a/README.md b/README.md index fd4ad4d..9f392b7 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,12 @@ A simple executor node for basic math operations in the Droq workflow engine, using the **dfx (Droqflow Executor)** framework. +![CI Status](https://github.com/droq/dfx-math-executor-node/workflows/Docker%20Build%20and%20Publish/badge.svg) +![Security Scan](https://github.com/droq/dfx-math-executor-node/workflows/Container%20Image%20Security%20Scan/badge.svg) +![License](https://img.shields.io/github/license/droq/dfx-math-executor-node) + +> **Note**: Replace `droq` with your GitHub organization name in the examples below. The CI/CD workflows automatically adapt to your organization. + ## dfx Framework The **dfx** framework is a standalone Python framework for building non-Langflow components. It provides: @@ -27,6 +33,67 @@ A simple component that multiplies two numbers. ## Running Locally +### Using Docker Compose (Recommended) + +The easiest way to run the project is with Docker Compose: + +```bash +# Clone the repository +git clone +cd dfx-math-executor-node + +# Configure ports (optional - edit .env file if needed) +# Default: HOST_PORT=8003 (host port), CONTAINER_PORT=8003 (container port) +# Edit .env file to customize ports and other settings + +# Build and run the service +docker compose up -d + +# Check status +docker compose ps + +# View logs +docker compose logs -f + +# Stop the service +docker compose down +``` + +#### Environment Variables + +Create or edit the `.env` file to customize the configuration: + +```bash +# Host port mapping (change the left side to customize the host port) +DFX_MATH_EXE_HOST_PORT=8003 + +# Container configuration +DFX_MATH_EXE_CONTAINER_PORT=8003 +DFX_MATH_EXE_HOST=0.0.0.0 + +# Logging level (DEBUG, INFO, WARNING, ERROR) +DFX_MATH_EXE_LOG_LEVEL=INFO + +# Container name +DFX_MATH_EXE_CONTAINER_NAME=droq-math-executor-node +``` + +**To use a different host port:** +```bash +# Edit .env file: DFX_MATH_EXE_HOST_PORT=8080 +# Then restart: docker compose down && docker compose up -d +``` + +### Required Permissions + +For the CI/CD to work with private registry, ensure: + +1. **Repository Settings**: Actions → General → "Read and write permissions" enabled +2. **Organization Settings**: Packages → "Allow public packages for this organization" enabled +3. **Team Access**: Grant team members access to private packages in organization settings + +### Without Docker (Local Development) + ```bash # Install dependencies uv sync @@ -41,6 +108,37 @@ uv run droq-math-executor-node 8003 - `GET /` - Service info - `POST /api/v1/execute` - Execute a component method +## Testing the Docker Compose Setup + +Once the service is running with `docker compose up -d`, you can test it: + +```bash +# Test health check +curl http://localhost:8003/health + +# Test service info +curl http://localhost:8003/ + +# Test multiplication (5 × 3 = 15) +curl -X POST http://localhost:8003/api/v1/execute \ + -H "Content-Type: application/json" \ + -d '{ + "component_state": { + "component_class": "DFXMultiplyComponent", + "component_module": "dfx.math.component.multiply", + "parameters": { + "number1": 5.0, + "number2": 3.0 + } + }, + "method_name": "multiply", + "is_async": false + }' + +# Run comprehensive tests +./test_api.sh +``` + ## Example Usage ```python @@ -60,14 +158,95 @@ POST /api/v1/execute } ``` +## CI/CD + +### Automated Builds + +This project uses GitHub Actions for automated CI/CD: + +- **Pull Requests**: Automated testing and Docker build validation +- **Main Branch**: Automated builds pushed to GitHub Container Registry +- **Tags**: Release builds with automatic GitHub releases +- **Security Scans**: Daily vulnerability scanning of container images + +### Docker Registry + +Images are automatically published to **GitHub Container Registry (GHCR)** as **private packages**: + +```bash +# Organization registry (replace with your organization name) +ghcr.io/droq/dfx-math-executor-node:latest +ghcr.io/droq/dfx-math-executor-node:v1.0.0 +ghcr.io/droq/dfx-math-executor-node:main +``` + +#### Private Registry Access + +Since images are published to a private registry, you'll need to authenticate: + +```bash +# Login to GitHub Container Registry +echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin + +# Or use a personal access token +docker login ghcr.io -u YOUR_USERNAME --password-stdin + +# Pull the private image +docker pull ghcr.io/droq/dfx-math-executor-node:latest +``` + +### Pulling from Registry + +```bash +# Pull latest version +docker pull ghcr.io/droq/dfx-math-executor-node:latest + +# Pull specific version +docker pull ghcr.io/droq/dfx-math-executor-node:v1.0.0 + +# Use in docker-compose.yml +services: + math-executor: + image: ghcr.io/droq/dfx-math-executor-node:latest + environment: + - HOST=0.0.0.0 + - PORT=8003 + - LOG_LEVEL=INFO + ports: + - "8003:8003" + restart: unless-stopped + +# Or use the .env file approach for easier configuration +# See Environment Variables section above +``` + +### Available Tags + +| Tag | Description | +|-----|-------------| +| `latest` | Latest stable release (main branch) | +| `main` | Latest build from main branch | +| `develop` | Latest build from develop branch | +| `vX.Y.Z` | Specific version releases | +| `main-` | Specific commits from main branch | + ## Architecture This executor node uses the **dfx** framework, which is: - **Standalone**: No dependency on `lfx` (Langflow framework) - **Lightweight**: Minimal dependencies (Pydantic, FastAPI, NATS) - **Compatible**: Works with the Droq workflow engine backend +- **Multi-platform**: Supports AMD64 and ARM64 architectures Components built with dfx can be: - Executed in isolated executor nodes - Registered in the Droq registry service - Discovered and used in workflows via the editor + +## Security + +- ✅ **Vulnerability Scanning**: Automated daily security scans +- ✅ **Dependency Reviews**: Automated dependency analysis on PRs +- ✅ **SBOM Generation**: Software Bill of Materials for each build +- ✅ **Non-root Container**: Runs as non-root user for security +- ✅ **Minimal Base Image**: Uses slim Python base image diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000..33792e1 --- /dev/null +++ b/compose.yml @@ -0,0 +1,18 @@ +services: + math-executor: + build: + context: . + dockerfile: Dockerfile + container_name: ${DFX_MATH_EXE_CONTAINER_NAME:-droq-math-executor-node} + environment: + - HOST=${DFX_MATH_EXE_HOST:-0.0.0.0} + - PORT=${DFX_MATH_EXE_CONTAINER_PORT:-8003} + - LOG_LEVEL=${DFX_MATH_EXE_LOG_LEVEL:-INFO} + ports: + - "${DFX_MATH_EXE_HOST_PORT:-8003}:${DFX_MATH_EXE_CONTAINER_PORT:-8003}" + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:${DFX_MATH_EXE_CONTAINER_PORT:-8003}/health"] + interval: 10s + timeout: 5s + retries: 3 \ No newline at end of file