diff --git a/.dockerignore b/.dockerignore index c3127dc1..5059a361 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,8 +7,10 @@ .gitignore .gitattributes -# CI/CD and GitHub-related files +# CI/CD and repo metadata ci/* +.github/* +.rabbit/ # macOS system files .DS_Store @@ -21,12 +23,12 @@ ci/* .vscode/ *.swp *.swo -.github/* -# Documentation files +# Documentation and development-only files docs/* README.md SECURITY.md +test/ # Local development scripts and helper files Makefile @@ -40,11 +42,12 @@ Makefile.variables *.bak *.old -# Ignore test configuration files +# Runtime defaults copied by Dockerfile live under src/configs. +# Keep the rest of src out of the image build context. src/* +!src/configs/ +!src/configs/** -# Ignore IDE specific and Prettier configuration files -.vscode/ +# Formatter/editor local files *.iml .prettierignore -.DS_Store diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml new file mode 100644 index 00000000..006465ff --- /dev/null +++ b/.github/workflows/docker-dependency-updater.yml @@ -0,0 +1,461 @@ +--- +name: udx-automation / dependency upgrade + +"on": + schedule: + - cron: "0 5 * * 1" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + DOCKERFILE: Dockerfile + PROBE_DOCKERFILE: .tmp/dependency-probe/Dockerfile + REPORT_PATH: docker-dependency-report.json + UPDATE_BRANCH: docker-dependency-updates + PR_TITLE: "chore(deps): docker dependency upgrade" + PR_AUTO_MERGE: "true" + PR_MERGE_METHOD: squash + COMMIT_MESSAGE: "chore(deps): update Docker dependency pins" + COPILOT_PROMPT_PARTS: >- + ci/prompts/docker-dependency-guardrails.md + ci/prompts/docker-dependency-apt.md + ci/prompts/docker-dependency-nonapt.md + ci/prompts/docker-dependency-output.md + +jobs: + config: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + + permissions: + contents: read + pull-requests: read + + outputs: + dockerfile: ${{ steps.config.outputs.dockerfile }} + existing_pr_found: ${{ steps.existing-pr.outputs.found }} + image: ${{ steps.config.outputs.image }} + prompt_parts: ${{ steps.config.outputs.prompt_parts }} + probe_dockerfile: ${{ steps.config.outputs.probe_dockerfile }} + report_path: ${{ steps.config.outputs.report_path }} + update_branch: ${{ steps.config.outputs.update_branch }} + pr_title: ${{ steps.config.outputs.pr_title }} + pr_auto_merge: ${{ steps.config.outputs.pr_auto_merge }} + pr_merge_method: ${{ steps.config.outputs.pr_merge_method }} + commit_message: ${{ steps.config.outputs.commit_message }} + should_run: ${{ steps.decision.outputs.should_run }} + skip_reason: ${{ steps.decision.outputs.skip_reason }} + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Load dependency updater defaults + id: config + run: | + set -euo pipefail + + { + echo "image=worker-deps-probe:${GITHUB_RUN_ID}" + echo "dockerfile=${DOCKERFILE}" + echo "prompt_parts=${COPILOT_PROMPT_PARTS}" + echo "probe_dockerfile=${PROBE_DOCKERFILE}" + echo "report_path=${REPORT_PATH}" + echo "update_branch=${UPDATE_BRANCH}" + echo "pr_title=${PR_TITLE}" + echo "pr_auto_merge=${PR_AUTO_MERGE}" + echo "pr_merge_method=${PR_MERGE_METHOD}" + echo "commit_message=${COMMIT_MESSAGE}" + } >> "${GITHUB_OUTPUT}" + + test -n "${DOCKERFILE}" + test -n "${COPILOT_PROMPT_PARTS}" + test -n "${PROBE_DOCKERFILE}" + test -f "${DOCKERFILE}" + for prompt_part in ${COPILOT_PROMPT_PARTS}; do + test -f "${prompt_part}" + done + echo "Dependency updater defaults loaded for ${DOCKERFILE}" + + - name: Stop if an update PR is already open + id: existing-pr + uses: actions/github-script@v8 + env: + UPDATE_BRANCH: ${{ steps.config.outputs.update_branch }} + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const head = `${owner}:${process.env.UPDATE_BRANCH}`; + + const prs = await github.rest.pulls.list({ + owner, + repo, + state: "open", + head, + per_page: 100 + }); + + core.setOutput("found", prs.data.length > 0 ? "true" : "false"); + if (prs.data.length > 0) { + console.log(`Update PR already open: ${prs.data[0].html_url}`); + } else { + console.log(`No open update PR found for ${head}`); + } + + - name: Decide updater execution + id: decision + run: | + set -euo pipefail + + if [ "${EXISTING_PR_FOUND}" = "true" ]; then + echo "should_run=false" >> "${GITHUB_OUTPUT}" + echo "skip_reason=Update PR already open for ${UPDATE_BRANCH}" >> "${GITHUB_OUTPUT}" + echo "Config decision: skip, update PR already open for ${UPDATE_BRANCH}" + exit 0 + fi + + echo "should_run=true" >> "${GITHUB_OUTPUT}" + echo "skip_reason=" >> "${GITHUB_OUTPUT}" + echo "Config decision: run upgrade, no open updater PR found." + env: + EXISTING_PR_FOUND: ${{ steps.existing-pr.outputs.found }} + UPDATE_BRANCH: ${{ steps.config.outputs.update_branch }} + + upgrade: + needs: config + if: needs.config.outputs.should_run == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + outputs: + apt_count: ${{ steps.probe.outputs.apt_count }} + changed: ${{ steps.changes.outputs.changed }} + pr_url: ${{ steps.create-pr.outputs.pull-request-url }} + + permissions: + contents: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + ref: ${{ github.ref }} + + - name: Generate no-pin apt probe report + id: probe + timeout-minutes: 8 + run: | + set -euo pipefail + + probe_dir="$(dirname "${PROBE_DOCKERFILE}")" + apt_packages_path="${probe_dir}/apt-packages.txt" + apt_versions_path="${probe_dir}/apt.tsv" + + mkdir -p "${probe_dir}" + + awk ' + { + line = $0 + + if (line ~ /apt-get install -y --no-install-recommends/) { + in_apt = 1 + } else if (in_apt) { + prefix = line + suffix = line + sub(/=.*/, "", prefix) + sub(/^[[:space:]]+[A-Za-z0-9.+-]+=[^[:space:]]+/, "", suffix) + line = prefix suffix + } + + print line + + if (in_apt && line ~ /&&[[:space:]]*\\?$/) { + in_apt = 0 + } + } + ' "${DOCKERFILE}" > "${PROBE_DOCKERFILE}" + + awk ' + /apt-get install -y --no-install-recommends/ { in_block=1; next } + in_block { + line=$0 + sub(/#.*/, "", line) + gsub(/\\/, "", line) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", line) + if (line ~ /^[[:alnum:].+-]+=/) { + sub(/=.*/, "", line) + print line + } + if ($0 ~ /&&[[:space:]]*\\?$/) { + in_block=0 + } + } + ' "${DOCKERFILE}" > "${apt_packages_path}" + + mapfile -t apt_packages < "${apt_packages_path}" + + docker build --pull --no-cache -f "${PROBE_DOCKERFILE}" -t "${PROBE_IMAGE}" . + docker run -i --rm --entrypoint bash "${PROBE_IMAGE}" -s -- "${apt_packages[@]}" > "${apt_versions_path}" <<'EOF' + set -euo pipefail + + dpkg-query -W -f='${binary:Package}\t${Version}\n' "$@" + EOF + + base_image="$(awk '$1 == "FROM" { print $2; exit }' "${DOCKERFILE}")" + + jq -n \ + --arg generated_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg base_image "${base_image}" \ + --arg dockerfile "${DOCKERFILE}" \ + --arg probe_dockerfile "${PROBE_DOCKERFILE}" \ + --rawfile apt_versions "${apt_versions_path}" \ + '{ + generated_at: $generated_at, + method: "no-pin apt probe", + base_image: $base_image, + strategy: { + summary: "Render a temporary Dockerfile with apt package pins removed, build it against the configured base image, then read installed versions with dpkg-query.", + dockerfile: $dockerfile, + probe_dockerfile: $probe_dockerfile + }, + dependencies: { + apt: ( + $apt_versions + | split("\n") + | map(select(length > 0)) + | map(split("\t") | {name: .[0], installed: .[1]}) + ) + } + }' > "${PROBE_REPORT}" + + apt_count="$(jq '.dependencies.apt | length' "${PROBE_REPORT}")" + echo "apt_count=${apt_count}" >> "${GITHUB_OUTPUT}" + echo "Upgrade probe: wrote ${PROBE_REPORT} with ${apt_count} apt packages" + env: + DOCKERFILE: ${{ needs.config.outputs.dockerfile }} + PROBE_DOCKERFILE: ${{ needs.config.outputs.probe_dockerfile }} + PROBE_IMAGE: ${{ needs.config.outputs.image }} + PROBE_REPORT: ${{ needs.config.outputs.report_path }} + + - name: Upload dependency report + uses: actions/upload-artifact@v5 + timeout-minutes: 3 + with: + name: docker-dependency-report + path: | + ${{ needs.config.outputs.report_path }} + ${{ needs.config.outputs.probe_dockerfile }} + + - name: Set up Node.js + uses: actions/setup-node@v6 + timeout-minutes: 3 + with: + node-version: "22" + + - name: Install Copilot CLI + timeout-minutes: 3 + run: | + npm install -g @github/copilot + echo "Upgrade setup: installed Copilot CLI" + + - name: Update Dockerfile with Copilot CLI + timeout-minutes: 8 + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_ALLOW_ALL: "true" + COPILOT_AUTO_UPDATE: "false" + PROMPT_PARTS: ${{ needs.config.outputs.prompt_parts }} + REPORT_PATH: ${{ needs.config.outputs.report_path }} + run: | + set -euo pipefail + + if [ -z "${COPILOT_GITHUB_TOKEN:-}" ]; then + echo "COPILOT_GITHUB_TOKEN secret is required. Use a fine-grained token with Copilot Requests permission." >&2 + exit 1 + fi + + prompt_path=".tmp/dependency-upgrade/copilot-prompt.md" + mkdir -p "$(dirname "${prompt_path}")" + : > "${prompt_path}" + for prompt_part in ${PROMPT_PARTS}; do + test -f "${prompt_part}" + { + cat "${prompt_part}" + echo + } >> "${prompt_path}" + done + + copilot \ + --prompt "$(cat "${prompt_path}")" \ + --allow-all-tools \ + --allow-all-urls \ + --no-ask-user \ + --no-auto-update \ + --silent \ + --share copilot-docker-dependency-session.md + + echo "Upgrade Copilot: completed; session saved to copilot-docker-dependency-session.md" + + - name: Guard Dockerfile-only changes + timeout-minutes: 1 + run: | + set -euo pipefail + + unexpected_changes="$(git diff --name-only | awk -v dockerfile="${DOCKERFILE}" '$0 != dockerfile')" + if [ -n "${unexpected_changes}" ]; then + echo "Upgrade guard: Copilot modified tracked files outside ${DOCKERFILE}:" >&2 + printf '%s\n' "${unexpected_changes}" >&2 + exit 1 + fi + + echo "Upgrade guard: tracked changes are limited to ${DOCKERFILE}" + env: + DOCKERFILE: ${{ needs.config.outputs.dockerfile }} + + - name: Detect changes + id: changes + shell: bash + run: | + set -euo pipefail + if git diff --quiet -- "${DOCKERFILE}"; then + echo "changed=false" >> "${GITHUB_OUTPUT}" + echo "Upgrade changes: no ${DOCKERFILE} changes" + else + echo "changed=true" >> "${GITHUB_OUTPUT}" + git diff -- "${DOCKERFILE}" > docker-dependency-update.diff + changed_lines="$(wc -l < docker-dependency-update.diff | tr -d ' ')" + echo "Upgrade changes: ${DOCKERFILE} changed; diff has ${changed_lines} lines" + fi + env: + DOCKERFILE: ${{ needs.config.outputs.dockerfile }} + + - name: Prepare pull request body + if: steps.changes.outputs.changed == 'true' + run: | + { + echo "## Summary" + echo + echo "Updates Dockerfile dependency pins using a no-pin apt probe and Copilot CLI." + echo + echo "## Evidence" + echo + echo "- Probe report artifact: \`${REPORT_PATH}\`" + echo "- Copilot session artifact: \`copilot-docker-dependency-session.md\`" + echo "- Build validation: handled by the repository Docker/CI workflows" + echo + echo "## Diff" + echo + echo '```diff' + sed -n '1,220p' docker-dependency-update.diff + echo '```' + } > docker-dependency-pr-body.md + echo "Upgrade PR body: prepared docker-dependency-pr-body.md" + env: + REPORT_PATH: ${{ needs.config.outputs.report_path }} + + - name: Upload Copilot session + uses: actions/upload-artifact@v5 + timeout-minutes: 3 + with: + name: copilot-docker-dependency-session + path: | + copilot-docker-dependency-session.md + docker-dependency-update.diff + if-no-files-found: ignore + + - name: Create pull request + id: create-pr + if: steps.changes.outputs.changed == 'true' + uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 + timeout-minutes: 5 + with: + commit-message: ${{ needs.config.outputs.commit_message }} + title: ${{ needs.config.outputs.pr_title }} + body-path: docker-dependency-pr-body.md + branch: ${{ needs.config.outputs.update_branch }} + add-paths: ${{ needs.config.outputs.dockerfile }} + draft: false + delete-branch: true + + - name: Enable pull request auto-merge + if: >- + steps.changes.outputs.changed == 'true' && + steps.create-pr.outputs.pull-request-url != '' && + needs.config.outputs.pr_auto_merge == 'true' + timeout-minutes: 3 + run: | + set -euo pipefail + + case "${MERGE_METHOD}" in + merge|rebase|squash) + if gh pr merge "${PR_URL}" --auto "--${MERGE_METHOD}"; then + echo "Upgrade auto-merge: enabled ${MERGE_METHOD} auto-merge for ${PR_URL}" + else + echo "Upgrade auto-merge: could not enable ${MERGE_METHOD} auto-merge for ${PR_URL}; leaving PR open or already mergeable." >&2 + fi + ;; + *) + echo "Unsupported merge method: ${MERGE_METHOD}" >&2 + exit 1 + ;; + esac + + env: + GH_TOKEN: ${{ github.token }} + MERGE_METHOD: ${{ needs.config.outputs.pr_merge_method }} + PR_URL: ${{ steps.create-pr.outputs.pull-request-url }} + + report: + needs: + - config + - upgrade + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + + steps: + - name: Write workflow summary + if: always() + run: | + pr_url="${PR_URL:-}" + if [ -z "${pr_url}" ]; then + pr_url="n/a" + fi + + skip_reason="${SKIP_REASON:-}" + if [ -z "${skip_reason}" ]; then + skip_reason="n/a" + fi + + apt_count="${APT_COUNT:-}" + if [ -z "${apt_count}" ]; then + apt_count="n/a" + fi + + changed="${CHANGED:-}" + if [ -z "${changed}" ]; then + changed="n/a" + fi + + { + echo "## udx-automation / dependency upgrade" + echo + echo "| Job | Result | Details |" + echo "| --- | --- | --- |" + echo "| config | ${{ needs.config.result }} | Dockerfile: \`${{ needs.config.outputs.dockerfile }}\`; branch: \`${{ needs.config.outputs.update_branch }}\`; auto-merge: \`${{ needs.config.outputs.pr_auto_merge }}\`; should run: \`${{ needs.config.outputs.should_run }}\` |" + echo "| upgrade | ${{ needs.upgrade.result }} | Report: \`${{ needs.config.outputs.report_path }}\`; apt: \`${apt_count}\`; Dockerfile changed: \`${changed}\`; PR: ${pr_url}; skip: \`${skip_reason}\` |" + echo "| report | ${{ job.status }} | Summary written |" + } >> "${GITHUB_STEP_SUMMARY}" + env: + APT_COUNT: ${{ needs.upgrade.outputs.apt_count }} + CHANGED: ${{ needs.upgrade.outputs.changed }} + PR_URL: ${{ needs.upgrade.outputs.pr_url }} + SKIP_REASON: ${{ needs.config.outputs.skip_reason }} diff --git a/.github/workflows/docker-ops.yml b/.github/workflows/docker-ops.yml index c099c85f..7fa23b49 100644 --- a/.github/workflows/docker-ops.yml +++ b/.github/workflows/docker-ops.yml @@ -7,12 +7,13 @@ name: Docker Ops - "**" paths: - ".github/workflows/docker-ops.yml" + - ".dockerignore" - "Dockerfile" - "bin/**" - "lib/**" - "src/**" - "etc/**" - - "tests/**" + - "test/**" - "Makefile" - "Makefile.variables" - "ci/**" diff --git a/.rabbit/context.yaml b/.rabbit/context.yaml new file mode 100644 index 00000000..80914ee1 --- /dev/null +++ b/.rabbit/context.yaml @@ -0,0 +1,101 @@ +# Generated by dev.kit repo — do not edit manually. +# Run `dev.kit repo` to refresh. +kind: repoContext +version: udx.dev/dev.kit/v1 +generator: + tool: dev.kit + repo: https://github.com/udx/dev.kit + version: 0.13.0 + generated_at: 2026-05-31T09:26:48Z + sources: + homepage: https://udx.dev/kit + repository: https://github.com/udx/dev.kit + package: https://www.npmjs.com/package/@udx/dev-kit + installation: https://github.com/udx/dev.kit/blob/latest/docs/installation.md + +repo: + name: worker + archetype: manifest-repo + +# Refs — Direct-read files and paths that define the repo contract. +# Note: Include only files or directories a repo consumer should read before code exploration. +# Note: Prefer README, focused docs, workflows, manifests, and explicit operational files. +# Note: Exclude broad implementation directories unless they are the contract themselves. + +refs: + - ./README.md + - ./Makefile + - ./docs/child-images.md + - ./docs/core-image.md + - ./src/configs/services.yaml + - ./src/configs/worker.yaml + - ./.github/workflows + - ./Dockerfile + - ./docs + +# Commands — Canonical repo entrypoints detected from strong repo signals. +# Note: Prefer declared make targets and package scripts before regex matches in docs. +# Note: Emit only commands that can be traced to a concrete source. +# Note: Record the source path so the command can be reviewed and corrected. + +commands: + verify: + run: make test + source: Makefile + build: + run: make build + source: Makefile + run: + run: make run + source: Makefile + +# Dependencies — Meaningful dependency-repo contracts such as reusable workflows, images, or versioned manifests this repo relies on. +# Note: Capture execution-shaping behavior defined outside the current checkout. +# Note: Avoid promoting standard package inventory or ordinary GitHub action refs into top-level context. +# Note: Normalize same-org versioned refs into repo slugs when possible. + +dependencies: + - repo: udx/reusable-workflows + kind: reusable workflow + resolved: true + archetype: workflow-repo + description: Reusable GitHub Actions workflow templates for CI/CD + used_by: + - .github/workflows/context7_sync.yml + - .github/workflows/docker-ops.yml + - repo: ubuntu:25.10 + kind: base image + resolved: false + used_by: + - Dockerfile + +# Manifests — YAML files that define repo-specific workflow, deploy, or contract behavior. +# Note: Include custom config/manifests that materially shape repo behavior or contract understanding. +# Note: Do not include workflow YAML only because it lives under .github/workflows. +# Note: Promote workflow files only when they declare reusable workflow refs or other repo-specific execution contracts. +# Note: Prefer structured kind and description metadata from the manifest itself. +# Note: Include hidden or nested contract dirs when they contain repo-owned manifests with meaningful metadata. + +manifests: + - path: .github/workflows/context7_sync.yml + kind: githubWorkflow + - path: .github/workflows/docker-ops.yml + kind: githubWorkflow + - path: src/configs/services.yaml + kind: workerService + declared_as: udx.io/worker-v1/service + source_repo: udx/worker + used_by: + - Dockerfile + evidence: + - version: udx.io/worker-v1/service + - path reference: Dockerfile + - path: src/configs/worker.yaml + kind: workerConfig + declared_as: udx.io/worker-v1/config + source_repo: udx/worker + used_by: + - Dockerfile + evidence: + - version: udx.io/worker-v1/config + - path reference: Dockerfile diff --git a/Dockerfile b/Dockerfile index 40cf4459..e9c4266b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,10 @@ FROM ubuntu:25.10 # Set the maintainer of the image LABEL maintainer="UDX CAG Team" -ARG AZURE_CLI_VERSION=2.85.0 -ARG PIP_VERSION=26.0.1 +ARG AZURE_CLI_VERSION=2.87.0 +ARG PIP_VERSION=26.1.2 ARG YQ_VERSION=4.53.2 -ARG GCLOUD_VERSION=565.0.0 +ARG GCLOUD_VERSION=571.0.0 # Set base environment variables ENV DEBIAN_FRONTEND=noninteractive \ @@ -43,18 +43,18 @@ USER root RUN apt-get update && \ apt-get install -y --no-install-recommends \ tzdata=2026a-0ubuntu0.25.10.1 \ - curl=8.14.1-2ubuntu1.2 \ + curl=8.14.1-2ubuntu1.3 \ bash=5.2.37-2ubuntu5 \ apt-utils=3.1.6ubuntu2 \ gettext=0.23.1-2build2 \ gnupg2=2.4.8-2ubuntu2.1 \ ca-certificates=20250419 \ lsb-release=12.1-1 \ - jq=1.8.1-3ubuntu1 \ + jq=1.8.1-3ubuntu1.1 \ zip=3.0-15ubuntu2 \ unzip=6.0-28ubuntu7 \ nano=8.4-1 \ - vim=2:9.1.0967-1ubuntu6.2 \ + vim=2:9.1.0967-1ubuntu6.5 \ python3.13=3.13.7-1ubuntu0.4 \ python3.13-venv=3.13.7-1ubuntu0.4 \ supervisor=4.2.5-3 && \ @@ -139,12 +139,13 @@ RUN mkdir -p \ # Create and set permissions for environment files touch ${WORKER_CONFIG_DIR}/environment && \ chown ${USER}:${USER} ${WORKER_CONFIG_DIR}/environment && \ - chmod 644 ${WORKER_CONFIG_DIR}/environment + chmod 600 ${WORKER_CONFIG_DIR}/environment # Copy worker files COPY bin/entrypoint.sh ${WORKER_BIN_DIR}/ COPY lib ${WORKER_LIB_DIR}/ -COPY etc/configs/worker/default.yaml ${WORKER_CONFIG_DIR}/worker.yaml +COPY src/configs/worker.yaml ${WORKER_CONFIG_DIR}/worker.yaml +COPY src/configs/services.yaml ${WORKER_CONFIG_DIR}/services.yaml COPY etc/configs/supervisor ${WORKER_CONFIG_DIR}/supervisor/ # Make scripts executable and initialize environment @@ -181,6 +182,7 @@ RUN \ find ${WORKER_BASE_DIR} ${WORKER_CONFIG_DIR} ${WORKER_LIB_DIR} ${WORKER_BIN_DIR} -type d -exec chmod 755 {} + && \ # Set base file permissions find ${WORKER_CONFIG_DIR} -type f -exec chmod 644 {} + && \ + chmod 600 ${WORKER_CONFIG_DIR}/environment && \ find ${WORKER_LIB_DIR} -type f ! -name process_manager.sh -exec chmod 644 {} + && \ # Make specific files executable chmod 755 \ diff --git a/Makefile b/Makefile index 6387c2d2..9dbb97c7 100644 --- a/Makefile +++ b/Makefile @@ -102,8 +102,8 @@ clean: test: clean @printf "$(COLOR_BLUE)$(SYM_ARROW) Running tests...$(COLOR_RESET)\n" @$(MAKE) run \ - VOLUMES="$(PWD)/src/tests:/home/udx/tests $(PWD)/src/examples/simple-config/.config/worker/worker.yaml:/home/udx/.config/worker/worker.yaml $(PWD)/src/examples/simple-service/.config/worker/services.yaml:/home/udx/.config/worker/services.yaml" \ - COMMAND="/home/udx/tests/main.sh" + VOLUMES="$(PWD)/test:/home/udx/test $(PWD)/src/examples/simple-config/.config/worker/worker.yaml:/home/udx/.config/worker/worker.yaml $(PWD)/src/examples/simple-service/.config/worker/services.yaml:/home/udx/.config/worker/services.yaml" \ + COMMAND="/home/udx/test/main.sh" @printf "$(COLOR_BLUE)$(SYM_ARROW) Following test output...$(COLOR_RESET)\n" @docker logs -f $(CONTAINER_NAME) & LOGS_PID=$$!; \ docker wait $(CONTAINER_NAME) > /dev/null; EXIT_CODE=$$?; \ diff --git a/README.md b/README.md index 885e73e9..0ec732a6 100644 --- a/README.md +++ b/README.md @@ -2,21 +2,23 @@ [![Docker Pulls](https://img.shields.io/docker/pulls/usabilitydynamics/udx-worker.svg)](https://hub.docker.com/r/usabilitydynamics/udx-worker) [![License](https://img.shields.io/github/license/udx/worker.svg)](LICENSE) [![Documentation](https://img.shields.io/badge/docs-udx.dev-blue.svg)](https://udx.dev/worker) -**Secure, containerized environment for DevSecOps automation** +Container runtime foundation for UDX automation images. -[Quick Start](#-quick-start) • [Documentation](#-documentation) • [Development](#️-development) • [Contributing](#-contributing) +[Quick Start](#quick-start) | [Documentation](#documentation) | [Development](#development) -## 🚀 Overview +## Overview -UDX Worker is a containerized solution that simplifies DevSecOps by providing: +UDX Worker is a base container image for automation workloads that need predictable runtime config, secret references, and process supervision. -- 🔒 **Secure Environment**: Built on zero-trust principles -- 🤖 **Automation Support**: Streamlined task execution -- 🔑 **Secret Management**: Automatic detection and resolution from multiple providers -- 📦 **12-Factor Compliance**: Modern application practices -- ♾️ **CI/CD Ready**: Seamless pipeline integration with environment-based overrides +It provides: -## 🏃 Quick Start +- `worker.yaml` for runtime env values, secret references, and opt-in runtime output. +- `services.yaml` for supervised processes inside the container. +- Secret reference resolution from AWS, Azure, and Google Cloud after provider auth exists. +- A shared CLI for inspecting and re-applying container runtime config. +- A stable base for child images that add workload-specific tools. + +## Quick Start ### Prerequisites @@ -63,7 +65,7 @@ docker run -d \ docker logs -f my-service ``` -### Example 2: Secrets Management with Authorization +### Example 2: Secret References ```bash # Define secrets configuration @@ -72,51 +74,34 @@ kind: workerConfig version: udx.io/worker-v1/config config: secrets: - API_KEY: "azure/key-vault/api-key" - DB_PASS: "aws/secrets/database" + API_KEY: "gcp/my-project/api-key" + DB_PASS: "aws/database-password/us-west-2" EOF -# Create base64-encoded Azure credentials -AZURE_CREDS=$(echo '{ - "client_id": "your-client-id", - "client_secret": "your-client-secret", - "tenant_id": "your-tenant-id" -}' | base64) - -# Run with cloud provider credentials +# Run with provider auth injected by the host/platform docker run -d \ --name my-secrets \ -v "$(pwd)/.config/worker:/home/udx/.config/worker" \ - -e AZURE_CREDS="${AZURE_CREDS}" \ usabilitydynamics/udx-worker:latest -# Verify authorization and secrets -docker exec my-secrets worker auth verify -docker exec my-secrets worker env get API_KEY +# Verify the resolved environment without printing the secret value +docker exec my-secrets sh -lc 'worker env show --filter API_KEY --format json | jq -e '\''has("API_KEY") and .API_KEY != ""'\'' >/dev/null' ``` -See [Authorization Guide](docs/authorization.md) for supported providers and credential formats (JSON, Base64, File Path). +See [Secrets](docs/secrets.md) for secret references and provider auth boundaries. -### 💡 Simplified Deployment +### Deployment -For easier deployment with automatic credential detection, use the [`@udx/worker-deployment`](https://www.npmjs.com/package/@udx/worker-deployment) CLI: +Deployment uses the host-native tool for the target environment. Mount runtime config into the container and pass provider credentials, workload identity, or secret references through the platform. ```bash -# Install -npm install -g @udx/worker-deployment - -# Generate config -worker config - -# Run with automatic GCP authentication -worker run +docker run --rm \ + -v "$(pwd)/.config/worker:/home/udx/.config/worker:ro" \ + -e API_KEY="gcp/my-project/api-key" \ + usabilitydynamics/udx-worker:latest ``` -Features: -- ✅ Auto-detects GCP credentials (service account keys, impersonation, workload identity) -- ✅ Zero-config for default file names -- ✅ Secure read-only mounts -- ✅ Interactive debugging mode +For Kubernetes, mount `worker.yaml` and `services.yaml` through ConfigMaps or Secrets and deploy the image with normal Kubernetes manifests. ### Development Setup @@ -138,22 +123,18 @@ make test More examples available in [src/examples/README.md](src/examples/README.md). -## 📚 Documentation +## Documentation -### Core Concepts -- [Docs Index](docs/index.md) - Start here -- [Runtime: Services](docs/runtime/services.md) - `services.yaml` -- [Runtime: Config](docs/runtime/config.md) - `worker.yaml` -- [Deployment](docs/deploy/README.md) - `deploy.yml` and `worker-deployment` -- [Authorization](docs/authorization.md) - Credential management -- [CLI Reference](docs/reference/cli.md) - Command line usage - -### Additional Resources -- [Container Structure](docs/reference/container-structure.md) - Directory layout -- [Development](docs/development/README.md) - Build, run, test, child images +- [CLI](docs/cli.md) - runtime inspection and re-apply commands +- [Config](docs/config.md) - `worker.yaml`, env values, runtime output +- [Secrets](docs/secrets.md) - secret references and provider auth boundaries +- [Services](docs/services.md) - `services.yaml` process config +- [Deployment](docs/deployment.md) - Docker, Kubernetes, and CI usage +- [Development](docs/development.md) - Build, test, and child image workflow +- [Reference Docs](docs/references/README.md) - provider auth options and container structure - [Examples](src/examples/README.md) - Runnable samples -## 🛠️ Development +## Development ```bash # Clone repository @@ -174,33 +155,18 @@ make test make help ``` -## 🤝 Contributing - -We welcome contributions! Here's how you can help: - -1. Fork the repository -2. Create a feature branch -3. Commit your changes -4. Push to your branch -5. Open a Pull Request - -Please ensure your PR: -- Follows our coding standards -- Includes appropriate tests -- Updates relevant documentation - -## 🔗 Resources +## Resources - [Docker Hub](https://hub.docker.com/r/usabilitydynamics/udx-worker) - [Documentation](https://udx.dev/worker) - [Product Page](https://udx.io/products/udx-worker) -## 🎯 Custom Development +## Custom Development Need specific features or customizations? [Contact our team](https://udx.io/) for professional development services. -## 📄 License +## License This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/bin/entrypoint.sh b/bin/entrypoint.sh index 64a94e64..1c4cb7e8 100644 --- a/bin/entrypoint.sh +++ b/bin/entrypoint.sh @@ -6,7 +6,14 @@ source "${WORKER_LIB_DIR}/utils.sh" log_info "Welcome to UDX Worker Container. Initializing environment..." # shellcheck disable=SC1091 -source "${WORKER_LIB_DIR}/environment.sh" +source "${WORKER_LIB_DIR}/worker_config.sh" +# shellcheck disable=SC1091 +source "${WORKER_LIB_DIR}/secrets.sh" +configure_environment || exit 1 + +# shellcheck disable=SC1091 +source "${WORKER_LIB_DIR}/runtime_output.sh" +emit_runtime_output || exit 1 # Start the process manager log_info "Starting process manager..." @@ -32,4 +39,4 @@ if [ "$#" -gt 0 ]; then fi # Keep the container running -wait \ No newline at end of file +wait diff --git a/ci/prompts/docker-dependency-apt.md b/ci/prompts/docker-dependency-apt.md new file mode 100644 index 00000000..92fa0508 --- /dev/null +++ b/ci/prompts/docker-dependency-apt.md @@ -0,0 +1,6 @@ +Apt dependency rules: +- The dependency report already resolves current apt package versions for the configured Ubuntu base image using a no-pin apt probe. +- The no-pin apt probe is authoritative for apt package updates: it was built from a temporary Dockerfile where apt pins were removed, then queried with `dpkg-query`. +- For apt packages, update Dockerfile pins only from `dependencies.apt[].installed` in the dependency report. +- Do not use apt websites, package search pages, or guessed versions for apt pins. +- If an apt package from Dockerfile is missing from the report, leave that package unchanged and explain it in the changelog. diff --git a/ci/prompts/docker-dependency-guardrails.md b/ci/prompts/docker-dependency-guardrails.md new file mode 100644 index 00000000..538b3f33 --- /dev/null +++ b/ci/prompts/docker-dependency-guardrails.md @@ -0,0 +1,14 @@ +You are updating Dockerfile dependency pins for this repository. + +Inputs: +- Dockerfile: `Dockerfile` +- Dependency report: `docker-dependency-report.json` + +Hard boundaries: +- This is an edit-only dependency update task. +- Read both input files before editing. +- Edit only Dockerfile dependency pins and ARG values. +- Do not edit workflow files, docs, tests, or application code. +- Do not validate, build, test, run the container pipeline, inspect GitHub Actions runs, wait for workflows, create pull requests, commit, push, or request reviews. +- Do not run `docker`, `make`, test commands, CI commands, `gh run`, `gh workflow`, `gh pr`, `git commit`, `git push`, or any command that waits on external workflow state. +- If an update cannot be verified from the dependency report or upstream release metadata without validation, leave it unchanged and mention why. diff --git a/ci/prompts/docker-dependency-nonapt.md b/ci/prompts/docker-dependency-nonapt.md new file mode 100644 index 00000000..e38d90d0 --- /dev/null +++ b/ci/prompts/docker-dependency-nonapt.md @@ -0,0 +1,7 @@ +Non-apt dependency rules: +- Detect ARG-pinned versions, URL-pinned versions, package-manager pins, and dynamically installed tools from Dockerfile. +- For each non-apt pinned dependency, identify its upstream source from Dockerfile context and check the latest stable version. +- Update only Dockerfile dependency pins and ARG values when the dependency report or upstream source shows a newer version. +- Preserve the ubuntu base image tag unless explicitly necessary to make reported apt versions valid. +- Keep dynamically installed dependencies unpinned unless Dockerfile already pins them; mention them in the changelog only. +- If a Dockerfile dependency cannot be checked confidently, leave it unchanged and mention the reason in the changelog. diff --git a/ci/prompts/docker-dependency-output.md b/ci/prompts/docker-dependency-output.md new file mode 100644 index 00000000..7c38fa2b --- /dev/null +++ b/ci/prompts/docker-dependency-output.md @@ -0,0 +1,20 @@ +Output: +- Print a changelog using this template: + +```text +Docker dependency changelog + +Updated: +- : -> () + +Unchanged: +- : () + +Observed but not pinned: +- : () + +Notes: +- +``` + +- Omit empty sections except `Notes`. diff --git a/deploy-gcp.yml b/deploy-gcp.yml deleted file mode 100644 index e5925a66..00000000 --- a/deploy-gcp.yml +++ /dev/null @@ -1,18 +0,0 @@ -# npm install -g @udx/worker-deployment -# worker run --config=deploy-gcp.yml - ---- -kind: workerDeployConfig -version: udx.io/worker-v1/deploy -config: - # Docker image to run - image: "usabilitydynamics/udx-worker:latest" - - env: - ACTORS_CLEANUP: "false" - - # Prefer command + args for special characters - command: "gcloud" - args: - - "auth" - - "list" diff --git a/deploy.yml b/deploy.yml deleted file mode 100644 index 4375754a..00000000 --- a/deploy.yml +++ /dev/null @@ -1,33 +0,0 @@ -# npm install -g @udx/worker-deployment -# worker config -# worker run - ---- -kind: workerDeployConfig -version: udx.io/worker-v1/deploy -config: - # Docker image to run - image: "usabilitydynamics/udx-worker:latest" - - env: - TEST_ENV_SECRET: "gcp/rabbit-ci-dev/worker-secret-test" - TEST_ENV_JSON_KEY: "gcp/rabbit-ci-dev/worker-secret-json-key" - - # Volume mounts (optional) - # Format: "host_path:container_path" or "host_path:container_path:ro" - # volumes: - # - "./worker.yaml:/home/udx/.config/worker/worker.yaml" - - # Ports to expose (optional) - # ports: - # - "80:80" - - # Command to run (optional - if not specified, uses container default) - # Tip: keep command to the executable and put complex values in args - # command: "/usr/local/bin/init.sh" - # args: - # - "--example-flag" - - # Service account impersonation (requires gcloud auth on host) - service_account: - email: "worker-site@rabbit-ci-dev.iam.gserviceaccount.com" diff --git a/docs/auth/README.md b/docs/auth/README.md deleted file mode 100644 index 17927107..00000000 --- a/docs/auth/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Provider Authentication Guides - -## Overview - -This directory contains detailed authentication documentation for each supported cloud provider. - -## When To Use - -Use these guides when you need provider-specific setup, credential formats, or CLI integration details. - -## Key Concepts - -- Each provider has its own auth flow and credential structure. -- Worker-deployment CLI can simplify authentication setup. - -## Examples - -- `docs/auth/gcp.md` - GCP authentication guide -- `docs/auth/azure.md` - Azure authentication guide (coming soon) -- `docs/auth/aws.md` - AWS authentication guide (coming soon) - -## Common Pitfalls - -- Mixing provider-specific formats. -- Passing plain secrets directly in config files. - -## Related Docs - -- `docs/authorization.md` -- `docs/runtime/config.md` -- `docs/reference/cli.md` diff --git a/docs/auth/aws.md b/docs/auth/aws.md deleted file mode 100644 index d5d7850a..00000000 --- a/docs/auth/aws.md +++ /dev/null @@ -1,14 +0,0 @@ -# AWS Authentication - -> **Coming Soon**: Detailed AWS authentication documentation - -## Quick Reference - -**Environment Variable:** `AWS_CREDS` - -**Supported Formats:** -- JSON -- Base64-encoded JSON -- File path - -For now, see the [general authorization guide](../authorization.md) for credential format examples. diff --git a/docs/auth/azure.md b/docs/auth/azure.md deleted file mode 100644 index 5bd2c822..00000000 --- a/docs/auth/azure.md +++ /dev/null @@ -1,24 +0,0 @@ -# Azure Authentication - -> **Coming Soon**: Detailed Azure authentication documentation - -## Quick Reference - -**Environment Variable:** `AZURE_CREDS` - -**Supported Formats:** -- JSON -- Base64-encoded JSON -- File path - -**Example:** -```json -{ - "client_id": "CLIENT_ID", - "client_secret": "CLIENT_SECRET", - "tenant_id": "TENANT_ID", - "subscription_id": "SUBSCRIPTION_ID" -} -``` - -For now, see the [general authorization guide](../authorization.md) for credential format examples. diff --git a/docs/auth/gcp.md b/docs/auth/gcp.md deleted file mode 100644 index 36b4ee9f..00000000 --- a/docs/auth/gcp.md +++ /dev/null @@ -1,189 +0,0 @@ -# GCP Authentication - -Google Cloud Platform supports multiple authentication methods, each suited for different use cases. - -## Authentication Methods - -### 1. Service Account Key (Most Common) - -Service account keys work for both local development and CI/CD environments. - -**JSON Format:** -```json -{ - "type": "service_account", - "project_id": "my-project-id", - "private_key_id": "key-id", - "private_key": "-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----\n", - "client_email": "my-sa@my-project.iam.gserviceaccount.com", - "client_id": "123456789", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/..." -} -``` - -**Usage:** -```bash -# Via environment variable -export GCP_CREDS='{"type":"service_account",...}' - -# Via file path -export GCP_CREDS="/path/to/service-account-key.json" - -# Via base64 encoding (recommended for CI/CD) -export GCP_CREDS=$(cat service-account-key.json | base64) -``` - -**Features:** -- ✅ Automatic `private_key` normalization (handles escaped newlines) -- ✅ Sets both `GOOGLE_APPLICATION_CREDENTIALS` and `GCP_CREDS` -- ✅ Authenticates with `gcloud` CLI -- ✅ Sets project automatically from `project_id` field - ---- - -### 2. Workload Identity Token (GitHub Actions / CI/CD) - -Keyless authentication using OIDC tokens - no service account keys needed! - -**JSON Format:** -```json -{ - "type": "external_account", - "audience": "//iam.googleapis.com/projects/PROJECT_NUMBER/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID", - "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", - "token_url": "https://sts.googleapis.com/v1/token", - "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/SA_EMAIL:generateAccessToken", - "credential_source": { - "file": "/path/to/token", - "format": { - "type": "text" - } - } -} -``` - -**GitHub Actions Example:** -```yaml -- uses: google-github-actions/auth@v3 - id: auth - with: - workload_identity_provider: ${{ secrets.WIF_PROVIDER }} - service_account: ${{ secrets.WIF_SERVICE_ACCOUNT }} - -- name: Run Worker - env: - GCP_CREDS: ${{ steps.auth.outputs.credentials_file_path }} - run: | - docker run -e GCP_CREDS usabilitydynamics/udx-worker:latest -``` - -**Features:** -- ✅ No long-lived credentials -- ✅ Automatic token refresh -- ✅ Works with Google Cloud client libraries -- ✅ Recommended for CI/CD pipelines - ---- - -### 3. Service Account Impersonation (Local Development) - -Use your personal gcloud credentials to impersonate a service account - no key files needed! - -**Setup:** -```bash -# 1. Authenticate with gcloud -gcloud auth login - -# 2. Set up Application Default Credentials (required for Terraform/SDKs) -gcloud auth application-default login - -# 3. Grant yourself impersonation permission -gcloud iam service-accounts add-iam-policy-binding \ - my-sa@my-project.iam.gserviceaccount.com \ - --member="user:$(gcloud config get-value account)" \ - --role="roles/iam.serviceAccountTokenCreator" \ - --project=MY_PROJECT -``` - -**Usage with worker-deployment CLI:** -```yaml -# deploy.yml -config: - service_account: - email: "my-sa@my-project.iam.gserviceaccount.com" - image: "usabilitydynamics/udx-worker:latest" - command: "worker run my-task" -``` - -```bash -# Run with automatic impersonation -worker run --config=deploy.yml -``` - -**Manual Docker Usage:** -```bash -# Generate impersonation credentials -gcloud auth application-default print-access-token > /tmp/token.txt - -# Run container with impersonation -docker run \ - -e GOOGLE_APPLICATION_CREDENTIALS=/home/udx/adc.json \ - -e CLOUDSDK_AUTH_ACCESS_TOKEN=$(cat /tmp/token.txt) \ - -v ~/.config/gcloud/application_default_credentials.json:/home/udx/adc.json:ro \ - usabilitydynamics/udx-worker:latest -``` - -**Features:** -- ✅ No service account key files -- ✅ Uses your personal credentials -- ✅ Temporary access tokens -- ✅ Easy permission management -- ✅ Works with Terraform, gcloud, and SDKs - -> **Note**: Impersonation bypasses the `gcp_authenticate()` function by setting `GOOGLE_APPLICATION_CREDENTIALS` and `CLOUDSDK_AUTH_ACCESS_TOKEN` directly. - ---- - -## Authentication Priority - -When multiple credential sources are available, the worker uses this priority: - -1. **`GOOGLE_APPLICATION_CREDENTIALS`** - If already set, skip authentication (used for impersonation) -2. **`GCP_CREDS`** - Process through `gcp_authenticate()` function: - - Detect credential type (service account key vs. workload identity token) - - Normalize service account keys (fix escaped newlines in `private_key`) - - Set `GOOGLE_APPLICATION_CREDENTIALS` - - Authenticate with `gcloud auth login --cred-file` - ---- - -## Using worker-deployment CLI - -The [`@udx/worker-deployment`](https://www.npmjs.com/package/@udx/worker-deployment) CLI simplifies GCP authentication: - -**Installation:** -```bash -npm install -g @udx/worker-deployment -``` - -**Quick Start:** -```bash -# Generate config template -worker config - -# Edit deploy.yml with your settings - -# Run with automatic credential detection -worker run -``` - -**Features:** -- ✅ Automatic credential detection (service account keys, impersonation, workload identity) -- ✅ Zero-config for default file names (`gcp-key.json`, `gcp-credentials.json`) -- ✅ Secure read-only mounts -- ✅ Support for custom credential paths - -See the [worker-deployment README](https://github.com/udx/worker-deployment) for detailed examples. diff --git a/docs/authorization.md b/docs/authorization.md deleted file mode 100644 index b6b4e28d..00000000 --- a/docs/authorization.md +++ /dev/null @@ -1,94 +0,0 @@ -# Worker Authorization - -## Overview - -The UDX Worker supports multiple cloud providers and services through environment-based credential management. - -## When To Use - -Use this when you need to: - -- Provide credentials to the worker container. -- Understand supported providers and formats. - -## Key Concepts - -- Credentials can be provided via env vars. -- Secrets can be JSON, Base64, or file paths. -- Secret references are resolved from provider paths in `worker.yaml` and matching runtime env vars. -- Authorization cleanup is controlled by `ACTORS_CLEANUP` (enabled by default). - -## Examples - -### Supported Providers - -| Provider | Environment Variable | Description | -| -------- | -------------------- | --------------------------------- | -| Azure | `AZURE_CREDS` | Azure cloud credentials | -| AWS | `AWS_CREDS` | Amazon Web Services credentials | -| GCP | `GCP_CREDS` | Google Cloud Platform credentials | - -### JSON Format - -```json -{ - "clientId": "CLIENT_ID", - "clientSecret": "CLIENT_SECRET", - "tenantId": "TENANT_ID", - "subscriptionId": "SUBSCRIPTION_ID" -} -``` - -### Base64 Format - -```bash -echo -n '{"clientId":"CLIENT_ID","clientSecret":"CLIENT_SECRET","tenantId":"TENANT_ID","subscriptionId":"SUBSCRIPTION_ID"}' | base64 -``` - -### File Path - -```bash -AZURE_CREDS="/path/to/azure_credentials.json" -``` - -### Provider-Specific Credential Keys - -- Azure (`AZURE_CREDS`): `clientId`, `clientSecret`, `tenantId`, `subscriptionId` -- AWS (`AWS_CREDS`): `AccessKeyId`, `SecretAccessKey`, optional `SessionToken` -- GCP (`GCP_CREDS`): standard GCP credential JSON (service account or other `gcloud --cred-file` compatible JSON) - -### Security Scenario: Long-Lived Credentials - -Example concern: -- A static service principal secret is stored in CI and reused for months. -- If leaked, an attacker can keep resolving secrets from the same vault scope until rotation. - -How worker design can mitigate: -- Fetch only referenced secrets at startup (for example `azure//`). -- Remove local auth artifacts after setup when `ACTORS_CLEANUP=true`. -- Override secret references per environment at deploy time for safer rotation workflows. - -How worker design can exacerbate: -- Resolved secrets are exported to process environment and can live for the container lifetime. -- Services can leak secrets if scripts print env vars or run with verbose shell tracing. -- A single broad credential can unlock multiple vault scopes in one worker instance. - -### Recommended Credential Posture - -- Prefer short-lived credentials or federation-based identity flows over long-lived static secrets. -- Grant least-privilege access to only the required secret scopes. -- Rotate provider credentials and secret values regularly. -- Split high-trust workloads into separate worker deployments when hard isolation is required. - -## Common Pitfalls - -- Using relative credential paths in production. -- Storing secrets in version control. -- Using long-lived provider credentials with broad access scope. -- Reusing one credential principal for unrelated services that require isolation. - -## Related Docs - -- `docs/runtime/config.md` -- `docs/deploy/README.md` -- `docs/auth/README.md` diff --git a/docs/development/child-images.md b/docs/child-images.md similarity index 62% rename from docs/development/child-images.md rename to docs/child-images.md index 7890eb23..287bd539 100644 --- a/docs/development/child-images.md +++ b/docs/child-images.md @@ -20,28 +20,16 @@ Avoid a child image when: ## Key Concepts - Child images extend `usabilitydynamics/udx-worker`. -- `worker gen` creates scaffolding to get started quickly. +- Child image scaffolding can be created manually with a Dockerfile that extends the base worker image. ## Examples -### Generate Scaffolding - -```bash -npm install -g @udx/worker-deployment - -# Generate a child image repo skeleton (dry-run + prompt) -worker gen repo - -# Generate a Dockerfile only (dry-run + prompt) -worker gen dockerfile -``` - ### Minimal Workflow -1. Generate a repo or Dockerfile. +1. Create a Dockerfile. 2. Add dependencies. 3. Build and tag the image. -4. Deploy using `deploy.yml`. +4. Run it with Docker, Kubernetes, or CI/CD. Example Dockerfile: @@ -56,21 +44,20 @@ Build: docker build -t my-org/udx-worker-custom:latest . ``` -Deploy (excerpt): +Run: -```yaml -kind: workerDeployConfig -version: udx.io/worker-v1/deploy -config: - image: "my-org/udx-worker-custom:latest" +```bash +docker run --rm \ + -v "$(pwd)/.config/worker:/home/udx/.config/worker:ro" \ + my-org/udx-worker-custom:latest ``` ## Common Pitfalls - Baking secrets into the image. -- Forgetting to update `deploy.yml` with the child image. +- Forgetting to update the host deployment image reference. ## Related Docs -- `docs/deploy/worker-deployment.md` -- `docs/reference/container-structure.md` +- `docs/deployment.md` +- `docs/references/container-structure.md` diff --git a/docs/reference/cli.md b/docs/cli.md similarity index 67% rename from docs/reference/cli.md rename to docs/cli.md index 58350d86..71b9f1f0 100644 --- a/docs/reference/cli.md +++ b/docs/cli.md @@ -14,17 +14,21 @@ Use the CLI when you need to: ## Key Concepts -- Commands are namespaced (`worker service`, `worker env`, `worker auth`). +- Commands are namespaced (`worker service`, `worker env`, `worker health`, `worker sbom`, `worker config`). - Most commands provide help when run without arguments. +- `worker env reload` and `worker config apply` rerun the same config/env/secret resolution path used by the entrypoint. ## Examples ```bash -# Show auth command help -worker auth - # Show service command help worker service + +# Inspect resolved environment +worker env status + +# Re-apply worker.yaml after provider auth has been established +worker env reload ``` ## Common Pitfalls @@ -34,5 +38,5 @@ worker service ## Related Docs -- `docs/runtime/services.md` -- `docs/runtime/config.md` +- `docs/services.md` +- `docs/config.md` diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 00000000..472d8205 --- /dev/null +++ b/docs/config.md @@ -0,0 +1,94 @@ +# Worker Config (`worker.yaml`) + +## Overview + +`worker.yaml` is the primary runtime configuration file. It defines environment variables, secret references, and opt-in runtime output used inside the worker container. + +## When To Use + +Use this when you need to: + +- Define runtime environment variables. +- Reference secrets that should be resolved at startup. +- Override defaults at runtime without rebuilding images. + +## Key Concepts + +- Runtime-only config: `/home/udx/.config/worker/worker.yaml`. +- Deployment env vars override `worker.yaml` values. +- Secret reference behavior is documented in `docs/secrets.md`. +- Provider authentication is not configured in `worker.yaml`. + +## Examples + +### Basic + +```yaml +kind: workerConfig +version: udx.io/worker-v1/config +config: + env: + APP_MODE: "worker" + AWS_REGION: "us-west-2" + secrets: + DB_PASSWORD: "aws/db-password/us-west-2" + API_KEY: "azure/kv-prod/api-key" +``` + +### Static Env and Secret References + +```yaml +kind: workerConfig +version: udx.io/worker-v1/config +config: + env: + API_KEY: "dev-only-static-key" + secrets: + DB_PASSWORD: "azure/kv-prod/db-password" +``` + +`API_KEY` is injected as-is. `DB_PASSWORD` is resolved from the provider after auth exists and then exported as an environment variable. + +### Runtime Environment and Precedence + +1. **Deployment environment variables** (highest priority) +2. Deployment environment variables containing secret references (resolved at startup) +3. `worker.yaml` `config.secrets` +4. `worker.yaml` `config.env` + +Example override: + +```yaml +# worker.yaml (production defaults) +config: + secrets: + ES_PASSWORD: "gcp/prod-project/es-password" +``` + +Deployment override example: + +```bash +docker run --rm \ + -e ES_PASSWORD="gcp/staging-project/es-password" \ + usabilitydynamics/udx-worker:latest +``` + +## Common Pitfalls + +- Storing plaintext secrets in `worker.yaml`. +- Putting cloud login/session setup in `worker.yaml`. +- Forgetting that deployment env vars override runtime config. + +## Runtime Output + +By default the worker does not print runtime config details or write output files. The entrypoint logs a short hint that output can be enabled. + +Set `WORKER_OUTPUT_FILE` when a deployment or workflow needs runtime config evidence. The worker writes redacted JSON runtime metadata to that path. + +Workflow-specific outputs such as `$GITHUB_OUTPUT`, `$GITHUB_STEP_SUMMARY`, or platform annotations should be generated by the workflow from the JSON file. + +## Related Docs + +- `docs/services.md` +- `docs/secrets.md` +- `docs/deployment.md` diff --git a/docs/development/core-image.md b/docs/core-image.md similarity index 84% rename from docs/development/core-image.md rename to docs/core-image.md index e889ebff..95c3e471 100644 --- a/docs/development/core-image.md +++ b/docs/core-image.md @@ -56,7 +56,7 @@ make log FOLLOW_LOGS=true make test ``` -The test target mounts `src/tests` and example configs into the container and runs `/home/udx/tests/main.sh`. +The test target mounts `test` and example configs into the container and runs `/home/udx/test/main.sh`. ## Common Pitfalls @@ -65,5 +65,5 @@ The test target mounts `src/tests` and example configs into the container and ru ## Related Docs -- `docs/runtime/services.md` -- `docs/runtime/config.md` +- `docs/services.md` +- `docs/config.md` diff --git a/docs/deploy/README.md b/docs/deploy/README.md deleted file mode 100644 index 59e96a94..00000000 --- a/docs/deploy/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Deployment - -## Overview - -Deployment is external to the worker container. This directory covers how to run the worker image consistently across local machines, CI/CD, and Kubernetes. - -## When To Use - -Use these docs when you need to: - -- Run the worker locally with a consistent config. -- Deploy in CI/CD (GitHub Actions, etc.). -- Mount runtime configs into Kubernetes. - -## Key Concepts - -- `deploy.yml` chooses the image, mounts runtime configs, and defines command/args. -- `worker.yaml` and `services.yaml` are runtime configs inside the container. - -## Examples - -- Local/CI usage: `docs/deploy/worker-deployment.md` -- CI image overrides: `docs/deploy/image-override.md` -- Kubernetes ConfigMaps: `docs/deploy/kubernetes.md` - -## Common Pitfalls - -- Editing runtime configs when you meant to change deployment behavior. -- Hardcoding image tags in CI instead of rendering `deploy.yml`. - -## Related Docs - -- `docs/runtime/config.md` -- `docs/runtime/services.md` diff --git a/docs/deploy/image-override.md b/docs/deploy/image-override.md deleted file mode 100644 index fe1c4e8f..00000000 --- a/docs/deploy/image-override.md +++ /dev/null @@ -1,58 +0,0 @@ -# CI/CD Image Override - -## Overview - -Override the worker image at deploy time by rendering a deploy template with an environment variable. - -## When To Use - -Use this when you: - -- Tag images per build and want to deploy that tag. -- Need to switch images without changing runtime configs. - -## Key Concepts - -- Keep `deploy.template.yml` in repo. -- Render to `deploy.yml` in CI. - -## Examples - -Template (`deploy.template.yml`): - -```yaml -kind: workerDeployConfig -version: udx.io/worker-v1/deploy -config: - image: "${WORKER_IMAGE}" - command: "echo" - args: - - "Hello from CI/CD" -``` - -Render + run: - -```bash -export WORKER_IMAGE="usabilitydynamics/udx-worker:latest" -envsubst < deploy.template.yml > deploy.yml -worker run --config=deploy.yml -``` - -If `envsubst` is unavailable: - -```bash -export WORKER_IMAGE="usabilitydynamics/udx-worker:latest" -sed "s|\${WORKER_IMAGE}|${WORKER_IMAGE}|g" deploy.template.yml > deploy.yml -worker run --config=deploy.yml -``` - -Example directory: `src/examples/deploy-image-override/` - -## Common Pitfalls - -- Committing generated `deploy.yml` with build-specific tags. -- Forgetting to set `WORKER_IMAGE` in CI. - -## Related Docs - -- `docs/deploy/worker-deployment.md` diff --git a/docs/deploy/kubernetes.md b/docs/deploy/kubernetes.md deleted file mode 100644 index 011c54f2..00000000 --- a/docs/deploy/kubernetes.md +++ /dev/null @@ -1,74 +0,0 @@ -# Kubernetes Deployment - -## Overview - -Kubernetes can mount `worker.yaml` and `services.yaml` into the container using ConfigMaps/Secrets. This keeps runtime configuration separate from the image. - -## When To Use - -Use this when you deploy the worker as a Kubernetes Deployment and want environment-specific configuration without rebuilding images. - -## Key Concepts - -- Store runtime configs as ConfigMaps (or Secrets for sensitive data). -- Mount into `/home/udx/.config/worker/`. - -## Examples - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: udx-worker-config -data: - worker.yaml: | - kind: workerConfig - version: udx.io/worker-v1/config - config: - env: - LOG_LEVEL: "info" - services.yaml: | - kind: workerService - version: udx.io/worker-v1/service - services: - - name: "logger" - command: "bash -c 'echo \"[startup]\"; while true; do echo \"[tick]\"; sleep 5; done'" - autostart: true - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: udx-worker -spec: - replicas: 1 - selector: - matchLabels: - app: udx-worker - template: - metadata: - labels: - app: udx-worker - spec: - containers: - - name: worker - image: usabilitydynamics/udx-worker:latest - volumeMounts: - - name: worker-config - mountPath: /home/udx/.config/worker - readOnly: true - volumes: - - name: worker-config - configMap: - name: udx-worker-config -``` - -## Common Pitfalls - -- Mounting configs to the wrong path. -- Putting secrets in ConfigMaps instead of Kubernetes Secrets. - -## Related Docs - -- `docs/runtime/config.md` -- `docs/runtime/services.md` diff --git a/docs/deploy/worker-deployment.md b/docs/deploy/worker-deployment.md deleted file mode 100644 index 47240118..00000000 --- a/docs/deploy/worker-deployment.md +++ /dev/null @@ -1,58 +0,0 @@ -# worker-deployment CLI - -## Overview - -`@udx/worker-deployment` standardizes how you run the worker image across different hosts and use cases. It keeps image selection, mounts, and runtime args in a single `deploy.yml` file. - -## When To Use - -Use `worker-deployment` when you want: - -- A consistent local run command (`worker run`). -- The same config to work in CI/CD runners. -- A portable deployment format across laptops and ephemeral hosts. - -## Key Concepts - -- `deploy.yml` is the deployment config. -- `worker.yaml` and `services.yaml` are runtime configs mounted into the container. - -## Examples - -### Quick Start - -```bash -npm install -g @udx/worker-deployment - -# Generate a template -worker config - -# Edit deploy.yml, then run -worker run -``` - -### Minimal Config - -```yaml -kind: workerDeployConfig -version: udx.io/worker-v1/deploy -config: - image: "usabilitydynamics/udx-worker:latest" - volumes: - - "./.config/worker:/home/udx/.config/worker:ro" - command: "echo" - args: - - "Hello from deploy.yml" -``` - -## Common Pitfalls - -- Forgetting to mount `worker.yaml`/`services.yaml` into the container. -- Putting runtime logic in `deploy.yml` instead of `services.yaml`. - -## Related Docs - -- `docs/deploy/README.md` -- `docs/deploy/image-override.md` -- `docs/runtime/config.md` -- `docs/runtime/services.md` diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 00000000..d39fb1da --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,54 @@ +# Deployment + +## Overview + +Deployment is external to the worker container. Use the host-native tool for the target environment: `docker run`, Docker Compose, `kubectl apply`, or the CI/CD platform's deployment step. + +## When To Use + +Use this when you need to: + +- Run the worker locally with Docker. +- Deploy in CI/CD (GitHub Actions, etc.). +- Mount runtime configs into a container runtime or orchestrator. + +## Key Concepts + +- Host tooling chooses the image, mounts runtime configs, and defines command/args. +- `worker.yaml` and `services.yaml` are runtime configs inside the container. +- Provider auth should be established by the host/platform or by the command running inside the container. +- Prefer native identity/env/token injection over custom credential volumes where the platform supports it. + +## Examples + +- Docker: + +```bash +docker run --rm \ + -v "$(pwd)/.config/worker:/home/udx/.config/worker:ro" \ + -e API_KEY="gcp/my-project/api-key" \ + usabilitydynamics/udx-worker:latest +``` + +- Docker with a child image: + +```bash +docker run --rm \ + -v "$(pwd)/.config/worker:/home/udx/.config/worker:ro" \ + my-org/udx-worker-custom:latest +``` + +For orchestrators such as Kubernetes, mount `worker.yaml` and `services.yaml` through the platform's normal config/secret primitives and keep deployment manifests outside the worker image. + +## Common Pitfalls + +- Editing runtime configs when you meant to change deployment behavior. +- Baking secrets into images instead of using env vars, mounted files, or Kubernetes Secrets. +- Putting runtime process definitions in host deployment config instead of `services.yaml`. +- Expecting worker deployment logic to create cloud sessions; deployment and auth are external concerns. + +## Related Docs + +- `docs/config.md` +- `docs/services.md` +- `docs/secrets.md` diff --git a/docs/development/README.md b/docs/development.md similarity index 75% rename from docs/development/README.md rename to docs/development.md index 8b09984a..afcc6b4a 100644 --- a/docs/development/README.md +++ b/docs/development.md @@ -17,10 +17,10 @@ Use these docs when you need to: - **Core image**: modify this repo when changing worker behavior. - **Child image**: extend the core image for extra dependencies. -## Examples +## Guides -- Core image workflow: `docs/development/core-image.md` -- Child image workflow: `docs/development/child-images.md` +- Core image workflow: `docs/core-image.md` +- Child image workflow: `docs/child-images.md` ## Common Pitfalls @@ -29,5 +29,5 @@ Use these docs when you need to: ## Related Docs -- `docs/deploy/README.md` -- `docs/reference/container-structure.md` +- `docs/deployment.md` +- `docs/references/container-structure.md` diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 76132df4..00000000 --- a/docs/index.md +++ /dev/null @@ -1,34 +0,0 @@ -# UDX Worker Documentation - -## Overview - -This documentation is organized by concern so it is clear what happens **inside** the worker container (runtime) vs what happens **outside** at deployment time. - -## When To Use - -Start here if you need to: - -- Configure runtime behavior (`services.yaml`, `worker.yaml`). -- Deploy the worker image across environments. -- Build the core image or child images. - -## Key Concepts - -- Runtime config lives inside the container. -- Deployment config selects the image and mounts runtime files. - -## Examples - -- Runtime: `docs/runtime/services.md` -- Deployment: `docs/deploy/README.md` -- Development: `docs/development/README.md` - -## Common Pitfalls - -- Mixing runtime config with deployment config. -- Editing the image when you only need a runtime change. - -## Related Docs - -- `docs/runtime/config.md` -- `docs/deploy/worker-deployment.md` diff --git a/docs/references/README.md b/docs/references/README.md new file mode 100644 index 00000000..7ad9476c --- /dev/null +++ b/docs/references/README.md @@ -0,0 +1,6 @@ +# Reference Docs + +Reference docs are stable lookup material for related platform context, best practices, and durable interpretation material. + +- [Cloud Providers Auth](cloud-providers-auth.md) +- [Container Structure](container-structure.md) diff --git a/docs/references/cloud-providers-auth.md b/docs/references/cloud-providers-auth.md new file mode 100644 index 00000000..df578b06 --- /dev/null +++ b/docs/references/cloud-providers-auth.md @@ -0,0 +1,27 @@ +# Cloud Providers Auth + +Cloud auth is intentionally outside the worker runtime. The worker only passes through provider environment variables/files and uses provider CLIs or SDK behavior after auth exists. + +## Options Matrix + +Prefer identity mechanisms that the runtime platform injects without worker-specific credential volumes. + +| Provider | Preferred path | File or volume fallback | Worker role | +|---|---|---|---| +| AWS | ECS task role, EKS Pod Identity, IRSA, or CI federation that exports standard AWS env/token variables | Shared AWS config/credentials files or projected web identity token files | Pass through AWS env/files and resolve secrets only after auth exists. | +| Azure | GitHub OIDC with `azure/login`, managed identity, or AKS workload identity | Azure CLI profile/config directory or projected federated token file when CLI tooling requires it | Pass through Azure env/files and resolve secrets only after auth exists. | +| Google Cloud | Attached service account, GKE/Cloud Run identity, or Workload Identity Federation | ADC or gcloud config files such as `GOOGLE_APPLICATION_CREDENTIALS` / `CLOUDSDK_CONFIG` | Pass through ADC/config and resolve secrets only after auth exists. | + +## Practical Rule + +Use the platform-native identity path first. Use mounted credential files only for local development, legacy tools, or provider CLIs that specifically require file-backed config. + +When auth happens inside the container, run the provider-native auth command first, then run: + +```bash +worker env reload +``` + +## Related Docs + +- `docs/secrets.md` diff --git a/docs/reference/container-structure.md b/docs/references/container-structure.md similarity index 96% rename from docs/reference/container-structure.md rename to docs/references/container-structure.md index 94385ab3..93fc67d2 100644 --- a/docs/reference/container-structure.md +++ b/docs/references/container-structure.md @@ -69,5 +69,5 @@ Python: ## Related Docs -- `docs/development/child-images.md` -- `docs/deploy/README.md` +- `docs/child-images.md` +- `docs/deployment.md` diff --git a/docs/runtime/config.md b/docs/runtime/config.md deleted file mode 100644 index c0c42af2..00000000 --- a/docs/runtime/config.md +++ /dev/null @@ -1,135 +0,0 @@ -# Worker Configuration (`worker.yaml`) - -## Overview - -`worker.yaml` is the primary runtime configuration file. It defines environment variables and secret references used **inside** the worker container. - -## When To Use - -Use this when you need to: - -- Define runtime environment variables. -- Reference secrets from cloud providers. -- Override defaults at runtime without rebuilding images. - -## Key Concepts - -- Runtime-only config: `/home/udx/.config/worker/worker.yaml`. -- Deployment env vars override `worker.yaml` values. -- Secret references use `provider//` format. -- Provider reference formats: - - `azure//` - - `gcp//` - - `aws//` - -## Examples - -### Basic - -```yaml -kind: workerConfig -version: udx.io/worker-v1/config -config: - env: - AZURE_CLIENT_ID: "12345678-1234-1234-1234-1234567890ab" - AWS_REGION: "us-west-2" - secrets: - DB_PASSWORD: "aws/db-password/us-west-2" - API_KEY: "azure/kv-prod/api-key" -``` - -### Static Secret Injection (`API_KEY`) and External Secret Resolution (`DB_PASSWORD`) - -```yaml -kind: workerConfig -version: udx.io/worker-v1/config -config: - env: - API_KEY: "dev-only-static-key" - secrets: - DB_PASSWORD: "azure/kv-prod/db-password" -``` - -`API_KEY` is injected as-is. `DB_PASSWORD` is resolved from the provider and then exported as an environment variable. - -### Secret References in `config.env` - -```yaml -config: - env: - DATABASE_URL: "gcp/my-project/db-connection-string" - API_TOKEN: "azure/kv-prod/api-token" - LOG_LEVEL: "info" -``` - -If an `env` value matches a secret reference format, the worker resolves it at startup. - -### Separate Secret Scopes for Different Services - -Use separate variable names in `worker.yaml`, then consume the right variable in each service: - -```yaml -# worker.yaml -kind: workerConfig -version: udx.io/worker-v1/config -config: - secrets: - SERVICE_A_DB_PASSWORD: "azure/kv-service-a/db-password" - SERVICE_B_DB_PASSWORD: "azure/kv-service-b/db-password" -``` - -```yaml -# services.yaml -kind: workerService -version: udx.io/worker-v1/service -services: - - name: "serviceA" - command: "bash -lc 'exec /home/udx/bin/service_a.sh'" - envs: - - "SERVICE_NAME=serviceA" - - - name: "serviceB" - command: "bash -lc 'exec /home/udx/bin/service_b.sh'" - envs: - - "SERVICE_NAME=serviceB" -``` - -`serviceA` should read `SERVICE_A_DB_PASSWORD`, and `serviceB` should read `SERVICE_B_DB_PASSWORD`. -For strict isolation boundaries, run separate worker instances with separate identities. - -### Runtime Environment and Precedence - -1. **Deployment environment variables** (highest priority) -2. Deployment environment variables containing secret references (resolved at startup) -3. `worker.yaml` `config.secrets` -4. `worker.yaml` `config.env` - -Example override: - -```yaml -# worker.yaml (production defaults) -config: - secrets: - ES_PASSWORD: "gcp/prod-project/es-password" -``` - -```yaml -# Kubernetes deployment (staging override) -spec: - containers: - - name: worker - env: - - name: ES_PASSWORD - value: "gcp/staging-project/es-password" -``` - -## Common Pitfalls - -- Storing plaintext secrets in `worker.yaml`. -- Forgetting that deployment env vars override runtime config. - -## Related Docs - -- `docs/runtime/services.md` -- `docs/deploy/README.md` -- `docs/deploy/kubernetes.md` diff --git a/docs/secrets.md b/docs/secrets.md new file mode 100644 index 00000000..8f9f8844 --- /dev/null +++ b/docs/secrets.md @@ -0,0 +1,90 @@ +# Secrets + +## Overview + +The worker resolves secret references from `worker.yaml` and environment variables after provider auth already exists. It does not log in to cloud providers or manage credential sessions. + +## When To Use + +Use this when you need to: + +- Define secret references in `worker.yaml`. +- Pass secret references through deployment environment variables. +- Re-run secret resolution after a command authenticates inside the container. + +## Key Concepts + +- Secret values are exported into the worker environment file and become available to services. +- `config.secrets` maps environment variable names to provider references. +- `config.env` values that look like provider references are resolved too. +- Deployment env vars override `worker.yaml` and can also contain secret references. +- Provider auth is external: Docker, Kubernetes, CI/CD, workload identity, mounted credentials, or a command inside the container owns login/session setup. + +Supported reference formats: + +- `azure//` +- `gcp//` +- `aws//` + +## Examples + +### `worker.yaml` + +```yaml +kind: workerConfig +version: udx.io/worker-v1/config +config: + secrets: + DB_PASSWORD: "azure/kv-prod/db-password" + API_KEY: "gcp/my-project/api-key" + env: + DATABASE_URL: "aws/database-url/us-west-2" +``` + +### Deployment Env Reference + +```bash +docker run --rm \ + -v "$(pwd)/.config/worker:/home/udx/.config/worker:ro" \ + -e API_KEY="gcp/my-project/api-key" \ + usabilitydynamics/udx-worker:latest +``` + +### Internal Auth Then Re-Resolve + +For development, testing, validation, or runbook workflows, authenticate with provider-native tooling inside the container and then rerun worker resolution: + +```bash +worker env reload +``` + +`worker config apply` is equivalent when the intent is to re-apply `worker.yaml`. + +This is still not a worker login feature. The auth command and credential storage remain owned by the user, child image, workflow, or runbook. + +### Resolve One Reference + +```bash +worker env resolve gcp/my-project/api-key +``` + +## Credential Posture + +- Prefer short-lived credentials, workload identity, or federation over static keys. +- Grant least-privilege access to only the required secret scopes. +- Pass only the provider env vars and mounted files needed by the workload. +- Mount credential directories read-only when files are required. +- Split high-trust workloads into separate worker deployments when isolation matters. + +## Common Pitfalls + +- Expecting built-in provider login commands. +- Storing plaintext secrets in `worker.yaml`. +- Reusing one broad credential principal for unrelated workloads. +- Adding custom credential volumes when platform identity or injected env/token files are available. + +## Related Docs + +- `docs/config.md` +- `docs/services.md` +- `docs/references/cloud-providers-auth.md` diff --git a/docs/runtime/services.md b/docs/services.md similarity index 91% rename from docs/runtime/services.md rename to docs/services.md index 2fae3e05..afab0211 100644 --- a/docs/runtime/services.md +++ b/docs/services.md @@ -1,4 +1,4 @@ -# Service Configuration (`services.yaml`) +# Service Config (`services.yaml`) ## Overview @@ -14,7 +14,7 @@ Use this when you need to: ## Key Concepts - Runtime-only: it lives inside the container at `/home/udx/.config/worker/`. -- Image selection happens at deployment time (see `docs/deploy/README.md`). +- Image selection happens at deployment time (see `docs/deployment.md`). - Each service is configured with a single `command` string (there is no `args` field in `services.yaml`). ## Examples @@ -115,12 +115,12 @@ services: ## Common Pitfalls -- Using `services.yaml` to select the image (use `deploy.yml` instead). +- Using `services.yaml` to select the image; image selection belongs to Docker, Kubernetes, or CI/CD deployment config. - Forgetting to mount `services.yaml` into the container. - Expecting an `args` field in `services.yaml` (put arguments directly in `command`). - Putting provider references (for example `azure/...`) in `services.yaml` `envs`. ## Related Docs -- `docs/runtime/config.md` -- `docs/deploy/README.md` +- `docs/config.md` +- `docs/deployment.md` diff --git a/etc/configs/worker/default.yaml b/etc/configs/worker/default.yaml deleted file mode 100644 index 5ec380b4..00000000 --- a/etc/configs/worker/default.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -kind: workerConfig -version: udx.io/worker-v1/config -config: - actors: - - type: gcp - creds: "${GCP_CREDS}" - - type: azure - creds: "${AZURE_CREDS}" - - type: aws - creds: "${AWS_CREDS}" diff --git a/lib/auth.sh b/lib/auth.sh deleted file mode 100644 index 33ab5db0..00000000 --- a/lib/auth.sh +++ /dev/null @@ -1,191 +0,0 @@ -#!/bin/bash - -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" - -# Array to track configured providers -declare -a configured_providers=() - -# Function to get env var names for a provider from actors JSON -get_provider_env_vars() { - local provider=$1 - local actors_json=$2 - - # Get all env var names from actor creds that match ${VAR} pattern - # shellcheck disable=SC2016 - echo "$actors_json" | jq -r ".[].creds" 2>/dev/null | \ - grep -o '\${[^}]*}' | sed 's/[\${}]//g' || true -} - -# Function to check if a provider is configured -is_provider_configured() { - local provider=$1 - local actors_json=$2 - - # Get provider's actors - local provider_actors - provider_actors=$(echo "$actors_json" | jq -r "[.[] | select(.type | startswith(\"$provider\"))]" 2>/dev/null) - - if [ -z "$provider_actors" ] || [ "$provider_actors" = "[]" ]; then - return 1 - fi - - # Get all possible env var names from actors - local env_vars - mapfile -t env_vars < <(get_provider_env_vars "$provider" "$provider_actors") - - # Check all possible env vars - for env_var in "${env_vars[@]}"; do - if [ -n "${!env_var}" ]; then - return 0 - fi - done - - # Check each actor's credentials - while IFS= read -r actor; do - [ -z "$actor" ] && continue - - local creds - creds=$(echo "$actor" | jq -r '.creds' 2>/dev/null) - [ "$creds" = "null" ] && continue - - # Evaluate creds as a reference to an environment variable - if [[ "$creds" =~ ^\$\{(.+)\}$ ]]; then - local env_var_name="${BASH_REMATCH[1]}" - creds="${!env_var_name}" - fi - - # If we find any valid credentials, return success - if [ -n "$creds" ]; then - return 0 - fi - done <<< "$(echo "$provider_actors" | jq -r '.[]')" - - return 1 -} - -# Function to authenticate actors -authenticate_actors() { - local actors_json="$1" - - if [[ -z "$actors_json" || "$actors_json" == "null" ]]; then - log_info "No worker actors found in the configuration." - return 0 - fi - - local actors_file - actors_file=$(mktemp) - echo "$actors_json" | jq -c '.[]' > "$actors_file" - - mapfile -t actors_array < "$actors_file" - rm -f "$actors_file" - - # Create local creds dir - log_info "Pre-creating local creds dir" - mkdir -p "$LOCAL_CREDS_DIR" - - for actor in "${actors_array[@]}"; do - local type provider creds auth_script auth_function - - type=$(resolve_env_vars "$(echo "$actor" | jq -r '.type')") - provider=$(echo "$type" | cut -d '-' -f 1) - creds=$(echo "$actor" | jq -r '.creds') - - # Evaluate creds as a reference to an environment variable - if [[ "$creds" =~ ^\$\{(.+)\}$ ]]; then - local env_var_name="${BASH_REMATCH[1]}" - creds="${!env_var_name}" - fi - - # Skip if the credentials are empty or not defined - if [[ -z "$creds" ]]; then - continue - else - log_success "Authentication" "Detected credentials for $type" - fi - - # Explicitly check for JSON format - if echo "$creds" | jq -e . >/dev/null 2>&1; then - log_info "Reading JSON credentials" - # Then, check if it's a file path - elif [[ -f "$creds" ]]; then - log_info "Reading credentials from file: $creds" - creds=$(cat "$creds") - # Finally, check if it's possibly base64 encoded - elif echo "$creds" | base64 --decode &>/dev/null && echo "$creds" | base64 --decode | jq empty &>/dev/null; then - log_info "Reading base64 encoded JSON credentials" - creds=$(echo "$creds" | base64 --decode) - else - log_error "Authentication" "Credentials format not recognized for $provider. Skipping..." - continue - fi - - # Proceed only if creds are valid JSON - if echo "$creds" | jq empty &>/dev/null; then - log_info "Processing credentials for $provider" - auth_script="${WORKER_LIB_DIR}/auth/${provider}.sh" - auth_function="${provider}_authenticate" - - if [[ -f "$auth_script" ]]; then - source "$auth_script" - - if command -v "$auth_function" > /dev/null; then - - if ! authenticate_provider "$provider" "$auth_function" "$creds"; then - log_error "Authentication" "Authentication failed for provider $provider." - return 1 - fi - configured_providers+=("$provider") - else - log_error "Authentication" "Authentication function $auth_function not found for $provider. Skipping..." - continue - fi - else - log_error "Authentication" "Authentication script $auth_script not found for $provider. Skipping..." - continue - fi - else - log_error "Authentication" "Invalid JSON credentials for $provider. Skipping..." - continue - fi - done - - if [[ ${#configured_providers[@]} -eq 0 ]]; then - log_info "No providers creds detected." - fi - - return 0 -} - -# Function to handle provider-specific authentication -authenticate_provider() { - local provider="$1" - local auth_function="$2" - local creds="$3" - local temp_config_file - - # Save the credentials data to a temporary file - temp_config_file=$(mktemp /tmp/actor_creds.XXXXXX) - echo "$creds" > "$temp_config_file" - - # Ensure cleanup with a trap in case of unexpected exit - trap 'rm -f "$temp_config_file"' EXIT - - # Call the authentication function with the temp file - if ! $auth_function "$temp_config_file"; then - log_error "Authentication failed for provider $provider." - return 1 - fi - - # Clean up the temporary file - rm -f "$temp_config_file" - trap - EXIT - - # Set an environment variable to mark successful authorization - export "${provider^^}_AUTHORIZED=true" - - return 0 -} - -# Example usage: -# authenticate_actors "$actors_json" \ No newline at end of file diff --git a/lib/auth/aws.sh b/lib/auth/aws.sh deleted file mode 100644 index e11c3271..00000000 --- a/lib/auth/aws.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" - -# Example usage of the function -# aws_authenticate "/path/to/your/aws_creds.json" - -# Function to authenticate AWS using IAM user credentials -aws_authenticate() { - local creds_json="$1" - - # Read the contents of the file - local creds_content - creds_content=$(cat "$creds_json") - - if [[ -z "$creds_content" ]]; then - log_error "AWS Authentication" "No AWS credentials provided." - return 1 - fi - - # Extract necessary fields from the JSON credentials - local accessKeyId secretAccessKey sessionToken - - accessKeyId=$(echo "$creds_content" | jq -r '.AccessKeyId') - secretAccessKey=$(echo "$creds_content" | jq -r '.SecretAccessKey') - sessionToken=$(echo "$creds_content" | jq -r '.SessionToken') - - if [[ -z "$accessKeyId" || -z "$secretAccessKey" ]]; then - log_error "AWS Authentication" "Missing required AWS credentials." - return 1 - fi - - # Export the credentials as environment variables - export AWS_ACCESS_KEY_ID="$accessKeyId" - export AWS_SECRET_ACCESS_KEY="$secretAccessKey" - if [[ -n "$sessionToken" ]]; then - export AWS_SESSION_TOKEN="$sessionToken" - fi - - log_success "AWS Authentication" "AWS credentials set successfully." -} \ No newline at end of file diff --git a/lib/auth/azure.sh b/lib/auth/azure.sh deleted file mode 100644 index e913180d..00000000 --- a/lib/auth/azure.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# Function to authenticate Azure accounts -# -# Example usage of the function -# azure_authenticate "/path/to/your/azure_creds.json" - -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" - -# Function to authenticate Azure accounts -azure_authenticate() { - local creds_json="$1" - - # Read the contents of the file - local creds_content - creds_content=$(cat "$creds_json") - - if [[ -z "$creds_content" ]]; then - log_error "Azure Authentication" "No Azure credentials provided." - return 1 - fi - - # Extract necessary fields from the JSON credentials - local clientId clientSecret subscriptionId tenantId - - clientId=$(echo "$creds_content" | jq -r '.clientId') - clientSecret=$(echo "$creds_content" | jq -r '.clientSecret') - subscriptionId=$(echo "$creds_content" | jq -r '.subscriptionId') - tenantId=$(echo "$creds_content" | jq -r '.tenantId') - - if [[ -z "$clientId" || -z "$clientSecret" || -z "$subscriptionId" || -z "$tenantId" ]]; then - log_error "Azure Authentication" "Missing required Azure credentials." - return 1 - fi - - log_info "Authenticating Azure service principal..." - if ! az login --service-principal -u "$clientId" -p "$clientSecret" --tenant "$tenantId" >/dev/null 2>&1; then - log_error "Azure Authentication" "Azure service principal authentication failed." - return 1 - fi - - if ! az account set --subscription "$subscriptionId" >/dev/null 2>&1; then - log_error "Azure Authentication" "Failed to set Azure subscription." - return 1 - fi - - log_success "Azure Authentication" "Azure service principal authenticated and subscription set." -} diff --git a/lib/auth/gcp.sh b/lib/auth/gcp.sh deleted file mode 100644 index b5791cfa..00000000 --- a/lib/auth/gcp.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" - -# GCP Authentication Module -# Supports: Service Account Keys, Workload Identity Tokens -# All methods use: gcloud auth login --cred-file="$GOOGLE_APPLICATION_CREDENTIALS" -# -# Note: Impersonation is handled externally by setting GOOGLE_APPLICATION_CREDENTIALS -# and CLOUDSDK_AUTH_ACCESS_TOKEN directly, bypassing this module. -# -# Example usage: -# gcp_authenticate "/path/to/gcp_creds.json" -# gcp_authenticate "${GCP_CREDS}" - -# Function to authenticate with GCP -gcp_authenticate() { - local creds_json="$1" - - # Read the contents of the file - local creds_content - creds_content=$(cat "$creds_json") - - if [[ -z "$creds_content" ]]; then - log_error "GCP Authentication" "No GCP credentials provided." - return 1 - fi - - # If GOOGLE_APPLICATION_CREDENTIALS already set, do not override - if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then - log_info "GCP Authentication" "GOOGLE_APPLICATION_CREDENTIALS already set, skipping authentication." - return 0 - fi - - local creds_file="$LOCAL_CREDS_DIR/gcp_creds.json" - - # Check if this is a service account key that needs private_key normalization - if echo "$creds_content" | jq -e '.private_key' >/dev/null 2>&1; then - # Service account key - normalize private_key field - local clientEmail privateKey projectId - - clientEmail=$(echo "$creds_content" | jq -r '.client_email') - privateKey=$(echo "$creds_content" | jq -r '.private_key') - projectId=$(echo "$creds_content" | jq -r '.project_id') - - # Normalize private_key: handle escaped newlines and spacing issues - privateKey=$(echo "$privateKey" | sed 's/\\n/\n/g' | sed 's/- /\n-/g' | sed 's/ -/-\n/g') - - # Create normalized JSON file - jq -n --arg clientEmail "$clientEmail" --arg privateKey "$privateKey" --arg projectId "$projectId" \ - '{type: "service_account", client_email: $clientEmail, private_key: $privateKey, project_id: $projectId}' > "$creds_file" - else - # Other credential types (workload identity, impersonation) - use as-is - echo "$creds_content" > "$creds_file" - fi - - # Set GOOGLE_APPLICATION_CREDENTIALS for all methods - export GOOGLE_APPLICATION_CREDENTIALS="$creds_file" - - # Set GCP_CREDS for backward compatibility - export GCP_CREDS="$creds_file" - - # Authenticate with gcloud (works for all credential types) - log_info "GCP Authentication" "Authenticating with gcloud..." - if ! gcloud auth login --cred-file="$GOOGLE_APPLICATION_CREDENTIALS" >/dev/null 2>&1; then - log_error "GCP Authentication" "Failed to authenticate with gcloud." - return 1 - fi - - # Extract and set project ID if available - local projectId - projectId=$(echo "$creds_content" | jq -r '.project_id // empty' 2>/dev/null) - - if [[ -n "$projectId" && "$projectId" != "null" ]]; then - if ! gcloud config set project "$projectId" >/dev/null 2>&1; then - log_error "GCP Authentication" "Failed to set GCP project: $projectId" - return 1 - fi - log_success "GCP Authentication" "Authenticated successfully. Project: $projectId" - else - log_success "GCP Authentication" "Authenticated successfully." - fi -} \ No newline at end of file diff --git a/lib/cleanup.sh b/lib/cleanup.sh deleted file mode 100644 index 87de9fdf..00000000 --- a/lib/cleanup.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash - -# Include worker config utilities first -# shellcheck source=/dev/null -source "${WORKER_LIB_DIR}/worker_config.sh" - -# shellcheck source=/dev/null -source "${WORKER_LIB_DIR}/utils.sh" - -# Generic function to clean up authentication for any provider -cleanup_provider() { - local provider=$1 - local logout_cmd=$2 - local list_cmd=$3 - local name=$4 - local cleaned_up=false - - # Check if the provider's CLI is available - if ! command -v "$provider" > /dev/null; then - return 0 # Skip silently if CLI is not available - fi - - # Check if there are active sessions/accounts to clean up - if ! eval "$list_cmd" > /dev/null 2>&1; then - return 0 # Skip silently if no active sessions are found - fi - - # Check if the provider's CLI returns "no credentials" to skip - if eval "$list_cmd" > /dev/null 2>&1 | grep -q "no credentials"; then - return 0 # Skip silently if no active sessions are found - fi - - # Check if the output of the list command is empty - if [ -z "$(eval "$list_cmd" 2> /dev/null)" ]; then - return 0 # Skip silently if no active sessions are found - fi - - log_info "Cleaning up $name authentication" - - # Run the logout command and capture any output or errors - local logout_output - logout_output=$(eval "$logout_cmd" 2>&1) - local logout_status=$? - - # Check if the logout was successful or if an expected error message was returned - if [[ $logout_status -ne 0 ]]; then - if echo "$logout_output" | grep -q -E "No credentials available to revoke|No active sessions|No active accounts"; then - log_info "No active $name credentials to revoke." - else - log_error "Cleanup" "Failed to log out of $name: $logout_output" - return 1 - fi - else - log_success "Cleanup" "$name authentication cleaned up successfully." - cleaned_up=true - fi - - # Return the cleanup status for summary reporting - if [[ "$cleaned_up" == true ]]; then - return 0 - else - return 1 - fi -} - -# Function to clean up credential files -cleanup_cred_files() { - local provider=$1 - local env_var_name="${provider^^}_CREDS" # Convert to uppercase - local creds_value="${!env_var_name}" - - # Skip if no credentials value found - if [[ -z "$creds_value" ]]; then - return 0 - fi - - # If the value is a file path and exists, remove it - if [[ -f "$creds_value" ]]; then - log_info "Removing credential file for $provider: $creds_value" - if rm -f "$creds_value"; then - log_success "Cleanup" "Removed credential file for $provider" - return 0 - else - log_error "Cleanup" "Failed to remove credential file for $provider" - return 1 - fi - fi - - return 0 -} - -# Function to clean up actors based on the providers configured during authentication -cleanup_actors() { - # Check if cleanup is enabled - if [[ "${ACTORS_CLEANUP,,}" != "true" ]]; then - log_info "Actors cleanup is disabled via ACTORS_CLEANUP environment variable" - return 0 - fi - - # Skip cleanup if no providers were configured during authentication - if [[ ${#configured_providers[@]} -eq 0 ]]; then - return 0 - fi - - log_info "Starting cleanup of actors" - - # Track if any actual cleanup was performed - local any_cleanup=false - - # Only clean up providers that were actually configured - for provider in "${configured_providers[@]}"; do - # First cleanup any credential files - if cleanup_cred_files "$provider"; then - any_cleanup=true - fi - - # Then cleanup provider sessions - case "$provider" in - azure) - if cleanup_provider "az" "az logout" "az account show" "Azure"; then - any_cleanup=true - fi - ;; - gcp) - if cleanup_provider "gcloud" "gcloud auth revoke --all && unset GOOGLE_APPLICATION_CREDENTIALS" "gcloud auth list" "GCP"; then - any_cleanup=true - fi - ;; - aws) - if cleanup_provider "aws" "aws sso logout" "aws sso list-accounts" "AWS"; then - any_cleanup=true - fi - ;; - *) - log_warn "Unsupported or unavailable actor type for cleanup: $provider" - ;; - esac - done - - # Log a summary if no cleanup actions were needed - if [[ "$any_cleanup" == false ]]; then - log_info "No active sessions found for any configured providers." - fi - - # Remove local copy creds dir - if [ -d "$LOCAL_CREDS_DIR" ]; then - log_info "Removing local copy creds dir" - rm -rf "$LOCAL_CREDS_DIR" - fi - - # Clear the configured providers array - configured_providers=() - - return 0 -} - -# Example usage -# cleanup_actors diff --git a/lib/cli.sh b/lib/cli.sh index 4ebaf35a..6045c124 100644 --- a/lib/cli.sh +++ b/lib/cli.sh @@ -87,7 +87,7 @@ EOF cat << EOF Run any command without arguments to see its detailed help and usage information. -For example: 'worker auth' will show auth command help. +For example: 'worker service' will show service command help. EOF } @@ -132,4 +132,4 @@ else log_error "CLI" "Unknown command: $command" echo "Run 'worker help' to see available commands." exit 1 -fi \ No newline at end of file +fi diff --git a/lib/cli/auth.sh b/lib/cli/auth.sh deleted file mode 100644 index 23cd77c2..00000000 --- a/lib/cli/auth.sh +++ /dev/null @@ -1,269 +0,0 @@ -#!/bin/bash - -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" -source "${WORKER_LIB_DIR}/auth.sh" - -# Show help for auth command -auth_help() { - cat << EOF -Manage authentication and credentials - -Usage: worker auth [command] [provider] [options] - -Available Commands: - status Show authentication status for all or specific provider - login Re-authenticate with provider(s) using available credentials - logout Log out from provider(s) - -Options: - --format Output format for status command (e.g. json) - -Examples: - worker auth status # Show status of all providers - worker auth status azure # Show status of Azure only - worker auth status --format json # Show status in JSON format - worker auth login # Re-auth all providers with available creds - worker auth login azure # Re-auth Azure only - worker auth logout # Log out from all providers -EOF -} - -# Description: Display authentication status for all cloud providers -# Example: worker auth status [--format json] -show_auth_status() { - local target_provider=$1 - local format=$2 - - if [ "$format" != "json" ]; then - log_info "Auth" "Checking authentication status..." - fi - - # Initialize JSON array if json format - local json_output="[" - - # Load config once and extract actors section - local config actors - config=$(load_and_parse_config) - actors=$(get_config_section "$config" "actors") - - # Function to check a specific provider - check_provider_status() { - local provider=$1 - local status="Not configured" - local types="" - - # Get provider's actors - local provider_actors - provider_actors=$(echo "$actors" | jq -r "[.[] | select(.type | startswith(\"$provider\"))]" 2>/dev/null) - - if [ -n "$provider_actors" ] && [ "$provider_actors" != "[]" ]; then - types=$(echo "$provider_actors" | jq -r '.[].type' 2>/dev/null | tr '\n' ' ') - - # First check if provider has credentials - if is_provider_configured "$provider" "$provider_actors"; then - # Then check if it's authenticated - if check_provider_auth "$provider"; then - status="Authenticated" - state="active" - else - status="Needs re-auth" - state="needs_reauth" - fi - else - status="Not configured" - state="missing_creds" - fi - else - status="Not configured" - state="not_configured" - fi - - # Output status based on format - if [ "$format" = "json" ]; then - [ -n "$json_output" ] && [ "$json_output" != "[" ] && json_output+="," - json_output+=$(jq -n \ - --arg provider "$provider" \ - --arg state "$state" \ - --arg status "$status" \ - --arg types "$types" \ - '{provider: $provider, state: $state, status: $status, types: $types}') - else - case "$state" in - "active") log_success "Auth" "$provider: $status" ;; - "needs_reauth") log_info "Auth" "$provider: $status" ;; - "missing_creds") log_warn "Auth" "$provider: $status" ;; - "not_configured") log_info "Auth" "$provider: $status" ;; - esac - fi - } - - # Check status for specific provider or all providers - if [ -n "$target_provider" ]; then - check_provider_status "$target_provider" - else - check_provider_status "aws" - check_provider_status "gcp" - check_provider_status "azure" - - fi - - # Close JSON array if json format - if [ "$format" = "json" ]; then - json_output+="]" - echo "$json_output" - fi -} - -# Login to provider(s) -login_provider() { - local target_provider=$1 - log_info "Auth" "Authenticating providers..." - - # Load config once and extract actors section - local config actors - config=$(load_and_parse_config) - actors=$(get_config_section "$config" "actors") - - if [[ -z "$actors" || "$actors" == "null" ]]; then - log_warn "Auth" "No providers found in configuration" - return 1 - fi - - # Filter actors by provider if specified - if [[ -n "$target_provider" ]]; then - actors=$(echo "$actors" | jq -r "[.[] | select(.type | startswith(\"$target_provider\"))]") - if [[ "$actors" == "[]" ]]; then - log_warn "Auth" "$target_provider: Not configured" - return 1 - fi - fi - - # Use the same authentication flow as entrypoint - if authenticate_actors "$actors"; then - log_success "Auth" "Authentication complete" - return 0 - else - log_warn "Auth" "No providers were authenticated" - return 1 - fi -} - -# Logout from provider(s) -logout_provider() { - local target_provider=$1 - log_info "Auth" "Logging out providers..." - - # Source cleanup utilities - source "${WORKER_LIB_DIR}/cleanup.sh" - - # Function to logout from a specific provider - do_provider_logout() { - local provider=$1 - - case "$provider" in - aws) - cleanup_provider "aws" "aws sso logout" "aws sso list-accounts" "AWS" - ;; - azure) - cleanup_provider "az" "az logout" "az account show" "Azure" - ;; - gcp) - cleanup_provider "gcloud" "gcloud auth revoke --all" "gcloud auth list" "GCP" - ;; - esac - } - - if [ -n "$target_provider" ]; then - do_provider_logout "$target_provider" - else - for provider in aws gcp azure; do - do_provider_logout "$provider" - done - fi -} - - - -# Check if a provider is currently authenticated -check_provider_auth() { - local provider=$1 - - case "$provider" in - aws) - if aws sts get-caller-identity &>/dev/null; then - return 0 - fi - ;; - azure) - # Azure CLI can return non-zero exit code even when it succeeds - # so we check if the output contains valid JSON - if output=$(az account show 2>/dev/null) && echo "$output" | jq empty &>/dev/null; then - return 0 - fi - ;; - gcp) - if gcloud auth list --format="value(account)" 2>/dev/null | grep -q .; then - return 0 - fi - ;; - esac - - return 1 -} - -# Handle auth commands -auth_handler() { - local cmd=$1 - shift - - case $cmd in - status) - local provider="" - local format="" - - # Parse arguments - while [ $# -gt 0 ]; do - case "$1" in - --format) - format="$2" - shift 2 - ;; - --*) - log_error "CLI" "Unknown option: $1" - return 1 - ;; - *) - if [ -z "$provider" ]; then - provider="$1" - else - log_error "CLI" "Unexpected argument: $1" - return 1 - fi - shift - ;; - esac - done - - show_auth_status "$provider" "$format" - return $? - ;; - login) - login_provider "$provider" - return $? - ;; - logout) - logout_provider "$provider" - return $? - ;; - "" | help) - auth_help - return 0 - ;; - *) - log_error "CLI" "Unknown command: auth" - auth_help - return 1 - ;; - esac -} diff --git a/lib/cli/config.sh b/lib/cli/config.sh index 24971cb2..270bd878 100644 --- a/lib/cli/config.sh +++ b/lib/cli/config.sh @@ -92,7 +92,6 @@ edit_config() { # Create file if it doesn't exist if [ ! -f "$config_file" ]; then - # Copy built-in config first to ensure actors section is preserved if [ -f "$BUILT_IN_CONFIG" ]; then cp "$BUILT_IN_CONFIG" "$config_file" || { log_error "Config" "Failed to copy built-in configuration" @@ -128,9 +127,9 @@ EOF show_locations() { cat << EOF Configuration Locations: - Built-in config: $BUILT_IN_CONFIG + Built-in config: $BUILT_IN_CONFIG User config: $USER_CONFIG - Merged config: $MERGED_CONFIG + Active config: $(get_worker_config_path) EOF } @@ -157,7 +156,6 @@ EOF if [ -f "$USER_CONFIG" ]; then log_success "Config" "Configuration initialized at $USER_CONFIG" - merge_worker_configs # Merge with built-in config return 0 else log_error "Config" "Failed to create configuration file" @@ -187,32 +185,12 @@ show_diff() { # Example: worker config apply apply_config() { log_info "Config" "Parsing and applying configuration..." - - # Load and parse the configuration - local config_json - if ! config_json=$(load_and_parse_config); then - log_error "Config" "Failed to load and parse configuration" - return 1 - fi - # Export variables from the configuration - if ! export_variables_from_config "$config_json"; then - log_error "Config" "Failed to export variables from configuration" + if ! configure_environment; then + log_error "Config" "Failed to parse and apply configuration" return 1 fi - # Extract secrets section from config - local secrets_json - secrets_json=$(echo "$config_json" | jq -r '.config.secrets // {}') - - # Fetch and set secrets if any are defined - if [[ "$secrets_json" != "{}" ]]; then - if ! fetch_secrets "$secrets_json"; then - log_error "Config" "Failed to fetch and set secrets" - return 1 - fi - fi - log_success "Config" "Configuration successfully parsed and applied" return 0 } diff --git a/lib/cli/env.sh b/lib/cli/env.sh index 7507f8b2..dfd37e3d 100644 --- a/lib/cli/env.sh +++ b/lib/cli/env.sh @@ -23,7 +23,7 @@ Available Commands: Options: --format Output format (text/json) --filter Filter variables by prefix - --include-secrets Include secrets in output (masked) + --include-secrets Include unmasked secrets in output Examples: worker env show # Show all environment variables @@ -39,56 +39,65 @@ EOF # Description: Display environment variables with optional filtering # Options: --format text|json, --filter PATTERN, --include-secrets # Example: worker env show --format json --filter AWS_* --include-secrets +is_secret_env_name() { + local name="$1" + local config + + config=$(load_and_parse_config) || return 1 + echo "$config" | jq -e --arg name "$name" --arg pattern "^(${SUPPORTED_SECRET_PROVIDERS})/.+/.+" ' + (.config.secrets // {} | has($name)) or + ((.config.env // {} | .[$name] // "" | tostring) | test($pattern)) + ' >/dev/null +} + +format_env_value_for_output() { + local name="$1" + local include_secrets="$2" + local value + + if [[ "$include_secrets" != "true" ]] && is_secret_env_name "$name"; then + printf '%s' '********' + return 0 + fi + + value=$(get_env_value "$name") || return 1 + printf '%s' "$value" +} + show_environment() { local format=${1:-text} - local filter=$2 + local filter=${2:-} local include_secrets=${3:-false} - + local names # Check if environment file exists if [ ! -f "$WORKER_ENV_FILE" ]; then log_error "Env" "Environment file not found" return 1 fi + + names=$(grep "^export " "$WORKER_ENV_FILE" | cut -d'=' -f1 | cut -d' ' -f2) case $format in json) - # Get variables and convert to JSON - local vars - if [ -n "$filter" ]; then - vars=$(grep "^export $filter" "$WORKER_ENV_FILE") - else - vars=$(grep "^export" "$WORKER_ENV_FILE") - fi - - # Convert to JSON - local json="{" - local first=true - while IFS= read -r line; do - if [[ $line =~ ^export[[:space:]]+([^=]+)=\"([^\"]*)\" ]]; then - if [ "$first" = true ]; then - first=false - else - json="$json," - fi - key=${BASH_REMATCH[1]} - value=${BASH_REMATCH[2]} - json="$json\"$key\":\"$value\"" + local json="{}" + while IFS= read -r name; do + # shellcheck disable=SC2053 # Env filters intentionally support globs like AWS_*. + if [[ -n "$name" && ( -z "$filter" || "$name" == $filter ) ]]; then + local value + value=$(format_env_value_for_output "$name" "$include_secrets") + json=$(echo "$json" | jq --arg key "$name" --arg value "$value" '. + {($key): $value}') fi - done <<< "$vars" - json="$json}" - if command -v jq >/dev/null 2>&1; then - echo "$json" | jq . - else - echo "$json" - fi + done <<< "$names" + echo "$json" | jq . ;; text) - if [ -n "$filter" ]; then - grep "^export $filter" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/\1=\2/' - else - grep "^export" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/\1=\2/' - fi + while IFS= read -r name; do + # shellcheck disable=SC2053 # Env filters intentionally support globs like AWS_*. + if [[ -n "$name" && ( -z "$filter" || "$name" == $filter ) ]]; then + printf '%s=%s\n' "$name" "$(format_env_value_for_output "$name" "$include_secrets")" + fi + done <<< "$names" ;; *) log_error "Env" "Unknown format: $format" @@ -119,13 +128,8 @@ set_environment() { return 1 fi - # Add to environment file if [ -f "$WORKER_ENV_FILE" ]; then - # Remove existing declaration if any - sed -i "/^export $name=/d" "$WORKER_ENV_FILE" - # Add new declaration - echo "export $name=\"$value\"" >> "$WORKER_ENV_FILE" - # Export in current session + upsert_env_value "$name" "$value" || return 1 export "$name=$value" log_success "Env" "Set $name to '$value'" else @@ -255,21 +259,14 @@ show_status() { local format=${1:-text} local env_file_exists=false - local secrets_file_exists=false local env_count=0 - local secrets_count=0 [ -f "$WORKER_ENV_FILE" ] && env_file_exists=true - [ -f "$WORKER_SECRETS_FILE" ] && secrets_file_exists=true if [ "$env_file_exists" = true ]; then env_count=$(grep -c "^export" "$WORKER_ENV_FILE" || echo 0) fi - if [ "$secrets_file_exists" = true ]; then - secrets_count=$(grep -c "^export" "$WORKER_SECRETS_FILE" || echo 0) - fi - case $format in json) { @@ -278,11 +275,6 @@ show_status() { echo " \"file\": \"$WORKER_ENV_FILE\"," echo " \"exists\": $env_file_exists," echo " \"variables\": $env_count" - echo " }," - echo " \"secrets\": {" - echo " \"file\": \"$WORKER_SECRETS_FILE\"," - echo " \"exists\": $secrets_file_exists," - echo " \"variables\": $secrets_count" echo " }" echo "}" } | jq '.' @@ -293,10 +285,6 @@ show_status() { echo "Environment File: $WORKER_ENV_FILE" echo " - Exists: $env_file_exists" echo " - Variables: $env_count" - echo - echo "Secrets File: $WORKER_SECRETS_FILE" - echo " - Exists: $secrets_file_exists" - echo " - Variables: $secrets_count" ;; *) log_error "Env" "Unknown format: $format" @@ -411,29 +399,11 @@ env_handler() { ;; reload) log_info "Env" "Reloading environment from configuration..." - local config_json - if ! config_json=$(load_and_parse_config); then - log_error "Env" "Failed to load and parse configuration" + if ! configure_environment; then + log_error "Env" "Failed to reload environment from configuration" return 1 fi - if ! export_variables_from_config "$config_json"; then - log_error "Env" "Failed to export variables from configuration" - return 1 - fi - - # Extract secrets section from config - local secrets_json - secrets_json=$(echo "$config_json" | jq -r '.config.secrets // {}') - - # Fetch and set secrets if any are defined - if [[ "$secrets_json" != "{}" ]]; then - if ! fetch_secrets "$secrets_json"; then - log_error "Env" "Failed to fetch and set secrets" - return 1 - fi - fi - log_success "Env" "Environment successfully reloaded from configuration" ;; status) @@ -451,4 +421,4 @@ env_handler() { exit 1 ;; esac -} \ No newline at end of file +} diff --git a/lib/cli/service.sh b/lib/cli/service.sh index 98b1af0d..16d55886 100644 --- a/lib/cli/service.sh +++ b/lib/cli/service.sh @@ -5,7 +5,23 @@ source "${WORKER_LIB_DIR}/utils.sh" # Constants SERVICES_CONFIG_DIR="${HOME}/.config/worker" -SERVICES_CONFIG_FILE="${SERVICES_CONFIG_DIR}/services.yaml" +USER_SERVICES_CONFIG_FILE="${SERVICES_CONFIG_DIR}/services.yaml" +BUILT_IN_SERVICES_CONFIG_FILE="${WORKER_CONFIG_DIR}/services.yaml" +SERVICES_CONFIG_FILE="" + +get_services_config_file() { + if [ -f "$USER_SERVICES_CONFIG_FILE" ] && [ -s "$USER_SERVICES_CONFIG_FILE" ]; then + echo "$USER_SERVICES_CONFIG_FILE" + return 0 + fi + + if [ -f "$BUILT_IN_SERVICES_CONFIG_FILE" ] && [ -s "$BUILT_IN_SERVICES_CONFIG_FILE" ]; then + echo "$BUILT_IN_SERVICES_CONFIG_FILE" + return 0 + fi + + return 1 +} # Show help for service command service_help() { @@ -66,9 +82,11 @@ service_handler() { return 0 fi + SERVICES_CONFIG_FILE=$(get_services_config_file) + # Check if services config exists before most commands if [ "$cmd" != "init" ]; then - if [ ! -f "$SERVICES_CONFIG_FILE" ]; then + if [ -z "$SERVICES_CONFIG_FILE" ]; then log_warn "Service" "No services configuration found" log_info "Service" "Run 'worker service' for information about service configuration" return 1 @@ -367,6 +385,29 @@ service_show_config() { fi } +# Description: Initialize a user service configuration +# Example: worker service init +init_service_config() { + mkdir -p "$SERVICES_CONFIG_DIR" || { + log_error "Service" "Failed to create service config directory: $SERVICES_CONFIG_DIR" + return 1 + } + + if [ -f "$USER_SERVICES_CONFIG_FILE" ]; then + log_info "Service" "Service configuration already exists at $USER_SERVICES_CONFIG_FILE" + return 0 + fi + + cat > "$USER_SERVICES_CONFIG_FILE" << 'EOF' +--- +kind: workerService +version: udx.io/worker-v1/service +services: [] +EOF + + log_success "Service" "Service configuration created at $USER_SERVICES_CONFIG_FILE" +} + # Description: Start, stop, or restart a service # Example: worker service restart my-app manage_service() { diff --git a/lib/env_handler.sh b/lib/env_handler.sh index d439c35d..1ad6d663 100644 --- a/lib/env_handler.sh +++ b/lib/env_handler.sh @@ -4,7 +4,64 @@ source "${WORKER_LIB_DIR}/utils.sh" # Environment file location -WORKER_ENV_FILE="/etc/worker/environment" +WORKER_ENV_FILE="${WORKER_ENV_FILE:-/etc/worker/environment}" + +ensure_env_file() { + local env_dir + env_dir=$(dirname "$WORKER_ENV_FILE") + + mkdir -p "$env_dir" || { + log_error "Environment" "Failed to create environment directory: $env_dir" + return 1 + } + + touch "$WORKER_ENV_FILE" || { + log_error "Environment" "Failed to create environment file: $WORKER_ENV_FILE" + return 1 + } + + chmod 600 "$WORKER_ENV_FILE" || { + log_error "Environment" "Failed to restrict environment file permissions: $WORKER_ENV_FILE" + return 1 + } +} + +upsert_env_value() { + local name="$1" + local value="$2" + + if [[ -z "$name" ]]; then + log_error "Environment" "Variable name not provided" + return 1 + fi + + if ! [[ "$name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then + log_error "Environment" "Invalid variable name: $name" + return 1 + fi + + ensure_env_file || return 1 + + local tmpfile + tmpfile=$(mktemp "${WORKER_ENV_FILE}.tmp.XXXXXX") || { + log_error "Environment" "Failed to create temporary environment file" + return 1 + } + + grep -v "^export $name=" "$WORKER_ENV_FILE" > "$tmpfile" || true + printf 'export %s=%q\n' "$name" "$value" >> "$tmpfile" + + mv "$tmpfile" "$WORKER_ENV_FILE" || { + rm -f "$tmpfile" + log_error "Environment" "Failed to update environment file" + return 1 + } + + chmod 600 "$WORKER_ENV_FILE" || { + log_error "Environment" "Failed to restrict environment file permissions: $WORKER_ENV_FILE" + return 1 + } +} # Generate environment file with regular variables generate_env_file() { @@ -18,37 +75,22 @@ generate_env_file() { log_info "Environment" "Loading environment variables from configuration" - # Create the environment file - touch "$WORKER_ENV_FILE" + ensure_env_file || return 1 - # Process each environment variable while IFS= read -r entry; do - # Extract key and value using string manipulation instead of regex - if [[ $entry == export* ]]; then - # Remove 'export ' prefix - local kv_pair=${entry#export } - # Extract key (everything before =) - local key=${kv_pair%%=*} - # Extract value (everything after = and remove quotes) - local value=${kv_pair#*=} - value=${value//\"/} - value=${value#\"} - value=${value%\"} - - # Check if the environment variable is exported (available in the environment) - # We use printenv to check if it's truly in the environment, not just a shell variable - if ! printenv "$key" > /dev/null 2>&1; then - # Variable doesn't exist in environment, add it from config - echo "export $key=\"$value\"" >> "$WORKER_ENV_FILE" - else - # Variable exists in environment, use that value instead - local env_value - env_value="$(printenv "$key")" - echo "export $key=\"$env_value\"" >> "$WORKER_ENV_FILE" - log_info "Environment" "Detected [$key] in container environment - using runtime value instead of config value" - fi + local key value + key=$(echo "$entry" | jq -r '.key') + value=$(echo "$entry" | jq -r '.value | tostring') + + if ! printenv "$key" > /dev/null 2>&1; then + upsert_env_value "$key" "$value" || return 1 + else + local env_value + env_value="$(printenv "$key")" + upsert_env_value "$key" "$env_value" || return 1 + log_info "Environment" "Detected [$key] in container environment - using runtime value instead of config value" fi - done < <(echo "$config" | yq eval '.config.env | to_entries | .[] | "export " + .key + "=\"" + .value + "\""' -) + done < <(echo "$config" | jq -c '.config.env // {} | to_entries[]') } # Internal function to resolve and append secrets @@ -65,18 +107,6 @@ _resolve_and_append_secrets() { return 1 fi - # Create a temporary file for resolved secrets - local temp_file - temp_file=$(mktemp) - - { - echo "" - echo "# Resolved Secrets" - echo "# Generated on $(date)" - echo "# DO NOT EDIT THIS FILE DIRECTLY" - echo - } > "$temp_file" - # Process each secret and resolve it while IFS= read -r secret; do local name value @@ -103,7 +133,7 @@ _resolve_and_append_secrets() { log_error "Environment" "Failed to resolve secret for $name" has_failures=true else - echo "export $name=\"$value\"" >> "$temp_file" + upsert_env_value "$name" "$value" || has_failures=true log_success "Environment" "Resolved secret for $name" fi done < <(echo "$secrets_json" | jq -c 'to_entries[]') @@ -111,14 +141,10 @@ _resolve_and_append_secrets() { # If any secret failed to resolve, error out if [[ "$has_failures" == "true" ]]; then log_error "Environment" "Failed to resolve one or more secrets" - rm -f "$temp_file" return 1 fi - # If we get here, all secrets resolved successfully - cat "$temp_file" >> "$WORKER_ENV_FILE" log_success "Environment" "Added all resolved secrets to environment file" - rm -f "$temp_file" } # Resolve secrets from worker.yaml config.secrets section @@ -133,6 +159,45 @@ resolve_env_var_secrets() { _resolve_and_append_secrets "$1" "false" } +# Configure environment from worker.yaml plus runtime secret references. +configure_environment() { + log_info "Starting environment configuration..." + + local resolved_config + resolved_config=$(load_and_parse_config) + if [[ -z "$resolved_config" ]]; then + log_error "Environment" "Configuration loading failed. Exiting..." + return 1 + fi + + if ! export_variables_from_config "$resolved_config"; then + log_error "Environment" "Failed to export variables." + return 1 + fi + + local secrets + secrets=$(get_config_section "$resolved_config" "secrets") + if [[ $? -eq 0 && -n "$secrets" && "$secrets" != "{}" ]]; then + log_info "Fetching secrets from configuration..." + if ! fetch_secrets "$secrets"; then + log_error "Environment" "Failed to fetch secrets." + return 1 + fi + else + log_info "No secrets defined in the configuration." + fi + + log_info "Checking for secret references in environment variables..." + if ! fetch_secrets_from_env_vars; then + log_error "Environment" "Failed to fetch secrets from environment variables." + return 1 + fi + + load_environment + + log_info "Secure environment setup completed successfully." +} + # Load environment variables and secrets load_environment() { if [ -f "$WORKER_ENV_FILE" ]; then @@ -181,17 +246,13 @@ format_env_vars() { # Initialize environment init_environment() { generate_env_file - generate_secrets_file load_environment - load_secrets } # Update environment when config changes update_environment() { generate_env_file - generate_secrets_file load_environment - load_secrets } # Get environment variable value @@ -204,11 +265,13 @@ get_env_value() { fi if [ -f "$WORKER_ENV_FILE" ]; then - grep "^export $var_name=" "$WORKER_ENV_FILE" | cut -d'=' -f2- | tr -d '"' - elif [ -f "$WORKER_SECRETS_FILE" ]; then - grep "^export $var_name=" "$WORKER_SECRETS_FILE" | cut -d'=' -f2- | tr -d '"' + ( + # shellcheck source=/dev/null + source "$WORKER_ENV_FILE" + printenv "$var_name" + ) else - log_error "Environment" "Neither environment nor secrets file exists" + log_error "Environment" "Environment file does not exist" return 1 fi } @@ -217,14 +280,13 @@ get_env_value() { list_env_vars() { local show_secrets="$1" local env_vars="" - local secret_vars="" if [ -f "$WORKER_ENV_FILE" ]; then env_vars=$(grep "^export" "$WORKER_ENV_FILE" | cut -d'=' -f1 | cut -d' ' -f2) fi - - if [ "$show_secrets" = "true" ] && [ -f "$WORKER_SECRETS_FILE" ]; then - secret_vars=$(grep "^export" "$WORKER_SECRETS_FILE" | cut -d'=' -f1 | cut -d' ' -f2) + + if [ "$show_secrets" = "true" ]; then + log_warn "Environment" "Secrets are stored in the worker environment file; separate secret listing is no longer used." fi if [ -n "$env_vars" ]; then @@ -232,9 +294,4 @@ list_env_vars() { echo "$env_vars" fi - if [ -n "$secret_vars" ]; then - echo - echo "Secret Variables:" - echo "$secret_vars" - fi -} \ No newline at end of file +} diff --git a/lib/environment.sh b/lib/environment.sh deleted file mode 100644 index 0da68d4a..00000000 --- a/lib/environment.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash - -# Include necessary modules -# shellcheck disable=SC1091 -source "${WORKER_LIB_DIR}/auth.sh" -# shellcheck disable=SC1091 -source "${WORKER_LIB_DIR}/secrets.sh" -# shellcheck disable=SC1091 -source "${WORKER_LIB_DIR}/cleanup.sh" -# shellcheck disable=SC1091 -source "${WORKER_LIB_DIR}/worker_config.sh" - -# shellcheck disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" - -# Main function to coordinate environment setup -configure_environment() { - log_info "Starting environment configuration..." - - # Load and resolve the worker configuration - local resolved_config - resolved_config=$(load_and_parse_config) - if [[ -z "$resolved_config" ]]; then - log_error "Environment" "Configuration loading failed. Exiting..." - return 1 - fi - - # Export variables from the configuration - if ! export_variables_from_config "$resolved_config"; then - log_error "Environment" "Failed to export variables." - return 1 - fi - - # Set default envs - if [[ -z "${ACTORS_CLEANUP:-}" ]]; then - export ACTORS_CLEANUP=true - fi - - if [[ -z "${LOCAL_CREDS_DIR:-}" ]]; then - export LOCAL_CREDS_DIR="$HOME/.config/worker/creds" - fi - - # Extract and authenticate actors - local actors - actors=$(get_config_section "$resolved_config" "actors") - if [[ $? -eq 0 && -n "$actors" ]]; then - log_info "Authenticating actors from configuration..." - if ! authenticate_actors "$actors"; then - log_error "Environment" "Failed to authenticate actors." - return 1 - fi - else - log_info "No actors defined in the configuration." - fi - - # Extract and fetch secrets - local secrets - secrets=$(get_config_section "$resolved_config" "secrets") - if [[ $? -eq 0 && -n "$secrets" ]]; then - log_info "Fetching secrets from configuration..." - if ! fetch_secrets "$secrets"; then - log_error "Environment" "Failed to fetch secrets." - return 1 - fi - else - log_info "No secrets defined in the configuration." - fi - - # Fetch secrets from environment variables with provider prefixes - log_info "Checking for secret references in environment variables..." - if ! fetch_secrets_from_env_vars; then - log_error "Environment" "Failed to fetch secrets from environment variables." - return 1 - fi - - # Perform cleanup - log_info "Cleaning up sensitive data..." - if ! cleanup_actors; then - log_error "Environment" "Failed to clean up actors." - return 1 - fi - - # Environment setup complete - log_info "Secure environment setup completed successfully." -} - -# Call the main function -configure_environment diff --git a/lib/process_manager.sh b/lib/process_manager.sh index d6d65ff9..a7e9ef93 100644 --- a/lib/process_manager.sh +++ b/lib/process_manager.sh @@ -5,7 +5,8 @@ source "${WORKER_LIB_DIR}/utils.sh" # Define paths USER_CONFIG_PATH="${HOME}/.config/worker/services.yaml" -CONFIG_FILE="${USER_CONFIG_PATH}" +BUILT_IN_CONFIG_PATH="${WORKER_CONFIG_DIR}/services.yaml" +CONFIG_FILE="" # Supervisor configuration paths COMMON_TEMPLATE_FILE="${WORKER_CONFIG_DIR}/supervisor/common.conf" @@ -16,15 +17,44 @@ FINAL_CONFIG="${WORKER_CONFIG_DIR}/supervisor/supervisord.conf" trap 'handle_supervisor_signals SIGTERM' SIGTERM trap 'handle_supervisor_signals SIGINT' SIGINT +ensure_process_manager_dependencies() { + local missing=false + + for command in yq jq; do + if ! command -v "$command" >/dev/null 2>&1; then + log_error "Process Manager" "$command is not installed. Please ensure it is available in the PATH." + missing=true + fi + done + + [[ "$missing" == "false" ]] +} + # Main execution main() { - # Check if user config exists - if [[ ! -f "${USER_CONFIG_PATH}" ]]; then - log_info "No services configuration found at ${USER_CONFIG_PATH}." + local enabled_services_count + + if ! ensure_process_manager_dependencies; then + exit 1 + fi + + CONFIG_FILE=$(get_service_config_path) + + if [[ -z "$CONFIG_FILE" ]]; then + log_info "No services configuration found." log_info "Run 'worker service' for information about service configuration" exit 0 fi + if ! enabled_services_count=$(count_enabled_services); then + exit 1 + fi + + if [[ "${enabled_services_count:-0}" -eq 0 ]]; then + log_info "No enabled services found in $CONFIG_FILE." + exit 0 + fi + log_info "Process Manager" "Starting process manager..." if ! configure_and_execute_services; then @@ -46,6 +76,39 @@ main() { wait } +get_service_config_path() { + if [[ -f "$USER_CONFIG_PATH" && -s "$USER_CONFIG_PATH" ]]; then + echo "$USER_CONFIG_PATH" + return 0 + fi + + if [[ -f "$BUILT_IN_CONFIG_PATH" && -s "$BUILT_IN_CONFIG_PATH" ]]; then + echo "$BUILT_IN_CONFIG_PATH" + return 0 + fi + + return 1 +} + +count_enabled_services() { + local enabled_services_count + + if ! enabled_services_count=$(yq e '.services // [] | map(select(.ignore != true)) | length' "$CONFIG_FILE" 2>/dev/null); then + log_error "Process Manager" "Failed to parse services configuration: $CONFIG_FILE" + return 1 + fi + + echo "${enabled_services_count:-0}" +} + +has_enabled_services() { + local enabled_services_count + + enabled_services_count=$(count_enabled_services) || return 1 + + [[ "${enabled_services_count:-0}" -gt 0 ]] +} + # Helper function to parse and process each service configuration parse_service_info() { local service_json="$1" @@ -210,14 +273,11 @@ start_supervisor() { # Function to check for service configurations should_generate_config() { - local enabled_services_count - # Extract services into JSON format - services_yaml=$(yq e -o=json '.services[] | select(.ignore != true)' "$CONFIG_FILE") - # Count the number of items in the JSON array, trimming any newlines or spaces - enabled_services_count=$(echo "$services_yaml" | jq -c '. | length' | tr -d '\n') + if [[ -z "$CONFIG_FILE" ]]; then + CONFIG_FILE=$(get_service_config_path) + fi - # Check if the configuration file exists and there is at least one enabled service - if [ -f "$CONFIG_FILE" ] && [ "${enabled_services_count:-0}" -gt 0 ]; then + if [ -f "$CONFIG_FILE" ] && has_enabled_services; then return 0 else return 1 @@ -226,6 +286,10 @@ should_generate_config() { # Function to configure services configure_services() { + if [[ -z "$CONFIG_FILE" ]]; then + CONFIG_FILE=$(get_service_config_path) + fi + if ! should_generate_config; then log_warn "Process Manager" "No services found in $CONFIG_FILE. No Supervisor configuration generated." return 1 @@ -265,4 +329,4 @@ configure_and_execute_services() { # Only run main if not in service mode if [ -z "$WORKER_SERVICE_MODE" ]; then main -fi \ No newline at end of file +fi diff --git a/lib/runtime_output.sh b/lib/runtime_output.sh new file mode 100644 index 00000000..736dc2d4 --- /dev/null +++ b/lib/runtime_output.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source "${WORKER_LIB_DIR}/utils.sh" +# shellcheck source=${WORKER_LIB_DIR}/worker_config.sh disable=SC1091 +source "${WORKER_LIB_DIR}/worker_config.sh" + +build_runtime_output_json() { + local config_json="$1" + local worker_config_path services_config_path env_json secrets_json + + worker_config_path=$(get_worker_config_path) + services_config_path="${HOME}/.config/worker/services.yaml" + if [[ ! -f "$services_config_path" && -f "${WORKER_CONFIG_DIR}/services.yaml" ]]; then + services_config_path="${WORKER_CONFIG_DIR}/services.yaml" + fi + + env_json=$(echo "$config_json" | jq '.config.env // {} | with_entries(.value = "redacted")' 2>/dev/null) || return 1 + secrets_json=$(echo "$config_json" | jq '.config.secrets // {} | with_entries(.value = "redacted")' 2>/dev/null) || return 1 + + jq -n \ + --arg worker_config_path "$worker_config_path" \ + --arg services_config_path "$services_config_path" \ + --arg worker_env_file "$WORKER_ENV_FILE" \ + --arg generated_at "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --argjson env "$env_json" \ + --argjson secrets "$secrets_json" \ + '{ + generated_at: $generated_at, + paths: { + worker_config: $worker_config_path, + services_config: $services_config_path, + environment: $worker_env_file + }, + env: $env, + secrets: $secrets, + secret_values: "redacted" + }' +} + +emit_runtime_output() { + local config_json runtime_json + + if [[ -z "${WORKER_OUTPUT_FILE:-}" ]]; then + log_info "Runtime output disabled. Set WORKER_OUTPUT_FILE to write redacted JSON runtime config for workflow/deployment integrations." + return 0 + fi + + config_json=$(load_and_parse_config) || return 1 + if ! runtime_json=$(build_runtime_output_json "$config_json"); then + log_error "Runtime output" "Failed to build runtime output JSON" + return 1 + fi + + mkdir -p "$(dirname "$WORKER_OUTPUT_FILE")" || return 1 + install -m 600 /dev/null "$WORKER_OUTPUT_FILE" || return 1 + printf '%s\n' "$runtime_json" > "$WORKER_OUTPUT_FILE" + log_info "Runtime output written to $WORKER_OUTPUT_FILE" +} diff --git a/lib/secrets.sh b/lib/secrets.sh index 6968fede..078b46c8 100644 --- a/lib/secrets.sh +++ b/lib/secrets.sh @@ -20,7 +20,6 @@ if [[ -z "${WORKER_INTERNAL_VARS+x}" ]]; then readonly WORKER_INTERNAL_VARS=( "AZURE_CONFIG_DIR" "AWS_CONFIG_FILE" - "GCP_CREDS" "TZ" ) fi diff --git a/lib/worker_config.sh b/lib/worker_config.sh index 274d96c4..7782418f 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -5,10 +5,9 @@ source "${WORKER_LIB_DIR}/utils.sh" # shellcheck source=${WORKER_LIB_DIR}/env_handler.sh disable=SC1091 source "${WORKER_LIB_DIR}/env_handler.sh" -# Paths for configurations -BUILT_IN_CONFIG="${WORKER_CONFIG_DIR}/worker.yaml" # Built-in default config -USER_CONFIG="${HOME}/.config/worker/worker.yaml" # Optional user config -MERGED_CONFIG="${WORKER_CONFIG_DIR}/worker.merged.yaml" # Result of merging both configs +# Paths for configuration. +BUILT_IN_CONFIG="${WORKER_CONFIG_DIR}/worker.yaml" +USER_CONFIG="${HOME}/.config/worker/worker.yaml" # Ensure `yq` is available if ! command -v yq >/dev/null 2>&1; then @@ -25,53 +24,34 @@ ensure_config_exists() { fi } -# Merge built-in and user-provided configurations -merge_worker_configs() { - # Ensure the merged configuration file exists - if [ ! -f "$MERGED_CONFIG" ]; then - touch "$MERGED_CONFIG" || { log_error "Worker configuration" "Failed to create merged configuration file at $MERGED_CONFIG"; return 1; } - fi - - # Ensure built-in config exists and has actors section - if ! ensure_config_exists "$BUILT_IN_CONFIG"; then - log_error "Worker configuration" "Built-in configuration not found" - return 1 +# Resolve the runtime configuration path. A mounted user config is preferred; +# otherwise the built-in default keeps the image runnable without mounts. +get_worker_config_path() { + if [[ -f "$USER_CONFIG" && -s "$USER_CONFIG" ]]; then + echo "$USER_CONFIG" + return 0 fi - # First copy built-in config (with actors) to merged config - if ! cp "$BUILT_IN_CONFIG" "$MERGED_CONFIG"; then - log_error "Worker configuration" "Failed to copy built-in configuration" - return 1 + if [[ -f "$BUILT_IN_CONFIG" && -s "$BUILT_IN_CONFIG" ]]; then + echo "$BUILT_IN_CONFIG" + return 0 fi - # If user config exists, merge env and secrets sections - if [[ -f "$USER_CONFIG" && -s "$USER_CONFIG" ]]; then - # Use yq to merge configs, preserving actors from built-in - if ! yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' "$MERGED_CONFIG" "$USER_CONFIG" > "${MERGED_CONFIG}.tmp"; then - log_error "Worker configuration" "Failed to merge configurations" - return 1 - fi - - # Check if merge was successful - if [ -s "${MERGED_CONFIG}.tmp" ]; then - mv "${MERGED_CONFIG}.tmp" "$MERGED_CONFIG" - return 0 - else - rm -f "${MERGED_CONFIG}.tmp" - log_error "Worker configuration" "Failed to merge configurations - empty result" - return 1 - fi - fi + echo "$BUILT_IN_CONFIG" } -# Load and parse the merged configuration +# Load and parse the active configuration. load_and_parse_config() { - merge_worker_configs || return 1 + local config_path + config_path=$(get_worker_config_path) + + if ! ensure_config_exists "$config_path"; then + return 1 + fi - # Parse the merged configuration into JSON local json_output - if ! json_output=$(yq eval -o=json "$MERGED_CONFIG" 2>/dev/null); then - log_error "Worker configuration" "Failed to parse merged YAML from $MERGED_CONFIG. yq returned an error." + if ! json_output=$(yq eval -o=json "$config_path" 2>/dev/null); then + log_error "Worker configuration" "Failed to parse YAML from $config_path. yq returned an error." return 1 fi @@ -119,4 +99,4 @@ get_config_section() { fi echo "$extracted_section" -} \ No newline at end of file +} diff --git a/src/configs/services.yaml b/src/configs/services.yaml new file mode 100644 index 00000000..6d999511 --- /dev/null +++ b/src/configs/services.yaml @@ -0,0 +1,4 @@ +--- +kind: workerService +version: udx.io/worker-v1/service +services: [] diff --git a/src/configs/worker.yaml b/src/configs/worker.yaml new file mode 100644 index 00000000..9358ed77 --- /dev/null +++ b/src/configs/worker.yaml @@ -0,0 +1,7 @@ +--- +kind: workerConfig +version: udx.io/worker-v1/config +config: + env: + WORKER_OUTPUT_FILE: "" + secrets: {} diff --git a/src/examples/README.md b/src/examples/README.md index 96b3b68e..789cb6cc 100644 --- a/src/examples/README.md +++ b/src/examples/README.md @@ -15,10 +15,3 @@ A set of small service scripts used to demonstrate supervisor behavior: Minimal `worker.yaml` used by tests and quick local runs: - `.config/worker/worker.yaml` - -## deploy-image-override - -Shows how to override the worker image in CI/CD using a deploy template: - -- `deploy.template.yml` -- `README.md` diff --git a/src/examples/deploy-image-override/README.md b/src/examples/deploy-image-override/README.md deleted file mode 100644 index 3f05a663..00000000 --- a/src/examples/deploy-image-override/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# Deploy Image Override (CI/CD) - -This example shows how to override the worker image at deploy time using an environment variable and a deploy template. - -Related docs: `docs/deploy/README.md` - -## Steps - -```bash -cd src/examples/deploy-image-override - -export WORKER_IMAGE="usabilitydynamics/udx-worker:latest" - -# Render deploy.yml from the template -envsubst < deploy.template.yml > deploy.yml - -# Run container using worker-deployment -worker run --config=deploy.yml -``` - -## Notes - -- `services.yaml` controls processes inside the container, not the container image. -- The image is selected in `deploy.yml` (worker-deployment). -- If `envsubst` is unavailable, use: - -```bash -sed "s|\${WORKER_IMAGE}|${WORKER_IMAGE}|g" deploy.template.yml > deploy.yml -``` diff --git a/src/examples/deploy-image-override/deploy.template.yml b/src/examples/deploy-image-override/deploy.template.yml deleted file mode 100644 index 87e7108b..00000000 --- a/src/examples/deploy-image-override/deploy.template.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -kind: workerDeployConfig -version: udx.io/worker-v1/deploy -config: - image: "${WORKER_IMAGE}" - command: "echo" - args: - - "Hello from CI/CD" diff --git a/src/examples/simple-service/README.md b/src/examples/simple-service/README.md index 44044644..1596add6 100644 --- a/src/examples/simple-service/README.md +++ b/src/examples/simple-service/README.md @@ -2,7 +2,7 @@ These scripts demonstrate common service behaviors for `services.yaml`. -Related docs: `docs/runtime/services.md` +Related docs: `docs/services.md` ## Scripts diff --git a/src/tests/modules/30_auth.sh b/src/tests/modules/30_auth.sh deleted file mode 100755 index ef011cba..00000000 --- a/src/tests/modules/30_auth.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Source test helpers -# shellcheck source=../test_helpers.sh disable=SC1091 -source "/home/udx/tests/test_helpers.sh" - -# Test authentication commands -print_header "Authentication Tests" - -# Test auth status -print_info "Testing: auth status" - -# Capture both stdout and stderr -AUTH_OUTPUT=$(worker auth status 2>&1) - -# Test that we got some output -if [ -z "$AUTH_OUTPUT" ]; then - print_error "auth status produced no output" - exit 1 -fi - -# Test that output contains provider status -if ! printf "%s" "$AUTH_OUTPUT" | grep -q "Auth:"; then - print_error "auth status should show provider status" - exit 1 -fi - -# Test auth status json format -print_info "Testing: auth status --format json" -AUTH_JSON=$(worker auth status --format json) -if ! echo "$AUTH_JSON" | jq -e '.[] | select(.provider == "aws") | .status' > /dev/null; then - print_error "auth status json should include provider status" - exit 1 -fi - -# All tests passed -print_success "All authentication tests passed" diff --git a/src/tests/main.sh b/test/main.sh similarity index 96% rename from src/tests/main.sh rename to test/main.sh index 88459e5a..b721bd12 100755 --- a/src/tests/main.sh +++ b/test/main.sh @@ -4,7 +4,7 @@ # Source test helpers # shellcheck source=./test_helpers.sh disable=SC1091 -source "/home/udx/tests/test_helpers.sh" +source "/home/udx/test/test_helpers.sh" # Exit on any error set -e @@ -30,7 +30,7 @@ PASSED=0 FAILED=0 # Directory containing test files -TEST_DIR="/home/udx/tests" +TEST_DIR="/home/udx/test" cd "$TEST_DIR" # Don't exit on test failures diff --git a/src/tests/modules/10_config.sh b/test/modules/10_config.sh similarity index 81% rename from src/tests/modules/10_config.sh rename to test/modules/10_config.sh index cb717822..91590c6c 100755 --- a/src/tests/modules/10_config.sh +++ b/test/modules/10_config.sh @@ -2,7 +2,7 @@ # Source test helpers # shellcheck source=../test_helpers.sh disable=SC1091 -source "/home/udx/tests/test_helpers.sh" +source "/home/udx/test/test_helpers.sh" # Test configuration commands print_header "Configuration Tests" @@ -30,5 +30,12 @@ if ! worker config locations | grep -q "/home/udx/.config/worker"; then exit 1 fi +# Test config apply +print_info "Testing: config apply" +if ! worker config apply; then + print_error "config apply should re-apply current configuration" + exit 1 +fi + # All tests passed print_success "All configuration tests passed" diff --git a/src/tests/modules/20_env.sh b/test/modules/20_env.sh similarity index 85% rename from src/tests/modules/20_env.sh rename to test/modules/20_env.sh index 31ec27f6..f967bfcb 100755 --- a/src/tests/modules/20_env.sh +++ b/test/modules/20_env.sh @@ -2,7 +2,7 @@ # Source test helpers # shellcheck source=../test_helpers.sh disable=SC1091 -source "/home/udx/tests/test_helpers.sh" +source "/home/udx/test/test_helpers.sh" # Test environment commands print_header "Environment Tests" @@ -43,5 +43,12 @@ for var in $CONFIG_ENV; do fi done +# Test environment reload +print_info "Testing: env reload" +if ! worker env reload; then + print_error "env reload should re-apply current configuration" + exit 1 +fi + # All tests passed print_success "All environment tests passed" diff --git a/src/tests/modules/40_service.sh b/test/modules/40_service.sh similarity index 97% rename from src/tests/modules/40_service.sh rename to test/modules/40_service.sh index 63112262..cac2d520 100755 --- a/src/tests/modules/40_service.sh +++ b/test/modules/40_service.sh @@ -2,7 +2,7 @@ # Source test helpers # shellcheck source=../test_helpers.sh disable=SC1091 -source "/home/udx/tests/test_helpers.sh" +source "/home/udx/test/test_helpers.sh" # Test service commands print_header "Service Tests" diff --git a/src/tests/modules/50_sbom.sh b/test/modules/50_sbom.sh similarity index 96% rename from src/tests/modules/50_sbom.sh rename to test/modules/50_sbom.sh index bf4c141b..2ee2683f 100755 --- a/src/tests/modules/50_sbom.sh +++ b/test/modules/50_sbom.sh @@ -2,7 +2,7 @@ # Source test helpers # shellcheck source=../test_helpers.sh disable=SC1091 -source "/home/udx/tests/test_helpers.sh" +source "/home/udx/test/test_helpers.sh" # Test SBOM commands print_header "SBOM Tests" diff --git a/src/tests/modules/60_health.sh b/test/modules/60_health.sh similarity index 96% rename from src/tests/modules/60_health.sh rename to test/modules/60_health.sh index 2ba2e12d..492cff2f 100755 --- a/src/tests/modules/60_health.sh +++ b/test/modules/60_health.sh @@ -2,7 +2,7 @@ # Source test helpers # shellcheck source=../test_helpers.sh disable=SC1091 -source "/home/udx/tests/test_helpers.sh" +source "/home/udx/test/test_helpers.sh" # Test health check commands print_header "Health Check Tests" diff --git a/src/tests/test_helpers.sh b/test/test_helpers.sh similarity index 100% rename from src/tests/test_helpers.sh rename to test/test_helpers.sh