From 5d08e90fd2b42f6cab9992269a8d6303933a3828 Mon Sep 17 00:00:00 2001 From: Mubai Hua Date: Sun, 19 Apr 2026 18:16:02 -0700 Subject: [PATCH 1/2] Improve local Docker dev bootstrap --- dev.sh | 74 ++++++++++++++++++- docker-compose.dev.yml | 10 +-- docs/SELF_HOSTING.md | 15 ++-- docs/agents/AGENT_RUNTIME.md | 7 +- docs/deployment/DEPLOYMENT.md | 8 +- .../commonly-bot/index.js | 58 +++++++++++---- 6 files changed, 142 insertions(+), 30 deletions(-) diff --git a/dev.sh b/dev.sh index dc981b16..f8e030fe 100755 --- a/dev.sh +++ b/dev.sh @@ -14,9 +14,78 @@ CLUSTER_NAME="${COMMONLY_CLUSTER_NAME:-commonly-local}" K8S_NAMESPACE="${COMMONLY_K8S_NAMESPACE:-commonly-local}" HELM_RELEASE="${COMMONLY_HELM_RELEASE:-commonly}" HELM_CHART="k8s/helm/commonly" +ENV_FILE=".env" +ENV_EXAMPLE_FILE=".env.example" + +get_env_value() { + local key="$1" + if [ -n "${!key:-}" ]; then + printf '%s' "${!key}" + return + fi + + if [ -f "$ENV_FILE" ]; then + local line + line=$(grep -E "^${key}=" "$ENV_FILE" | tail -n 1 || true) + if [ -n "$line" ]; then + printf '%s' "${line#*=}" + fi + fi +} + +has_nonempty_env_value() { + [ -n "$(get_env_value "$1")" ] +} + +ensure_local_env_file() { + if [ -f "$ENV_FILE" ]; then + return + fi + + if [ -f "$ENV_EXAMPLE_FILE" ]; then + cp "$ENV_EXAMPLE_FILE" "$ENV_FILE" + echo "📝 Created $ENV_FILE from $ENV_EXAMPLE_FILE" + echo " Review it before enabling optional integrations or AI features." + echo "" + return + fi + + echo "⚠️ $ENV_FILE is missing and $ENV_EXAMPLE_FILE was not found." + echo " Docker Compose may still start if your shell env already has the needed values." + echo "" +} + +print_local_dev_notes() { + echo "" + echo "Local dev notes:" + + if has_nonempty_env_value "GEMINI_API_KEY" \ + || has_nonempty_env_value "OPENAI_API_KEY" \ + || has_nonempty_env_value "OPENROUTER_API_KEY"; then + echo " • AI provider credentials detected." + else + echo " • No AI provider key configured. AI features will stay limited until you set one of:" + echo " GEMINI_API_KEY, OPENAI_API_KEY, or OPENROUTER_API_KEY in $ENV_FILE" + echo " Then run: ./dev.sh restart" + fi + + if has_nonempty_env_value "COMMONLY_SUMMARIZER_RUNTIME_TOKEN"; then + echo " • commonly-bot runtime token detected." + else + echo " • commonly-bot will stay idle until you provision it from Agents Hub." + echo " That is expected for fresh local setups." + fi + + if has_nonempty_env_value "CLAWDBOT_GATEWAY_TOKEN"; then + echo " • Clawdbot gateway token detected. Start it with: ./dev.sh clawdbot up" + else + echo " • Clawdbot services are optional. Add CLAWDBOT_GATEWAY_TOKEN in $ENV_FILE if you want local OpenClaw." + fi +} case "$1" in up) + ensure_local_env_file echo "🚀 Starting development environment..." # Set higher timeout for frontend npm install export COMPOSE_HTTP_TIMEOUT=300 @@ -28,6 +97,7 @@ case "$1" in echo "🐘 PostgreSQL: localhost:5432" echo "" echo "💡 To start Clawdbot: ./dev.sh clawdbot up" + print_local_dev_notes ;; down) @@ -37,9 +107,11 @@ case "$1" in ;; restart) + ensure_local_env_file echo "🔄 Restarting development environment..." docker-compose -f $COMPOSE_FILE restart echo "✅ Development environment restarted!" + print_local_dev_notes ;; logs) @@ -295,4 +367,4 @@ case "$1" in echo " ./dev.sh cluster test" echo " ./dev.sh cluster down" ;; -esac \ No newline at end of file +esac diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 1a9edb1b..2d5716d9 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,5 +1,3 @@ -version: '3' - services: # MongoDB service mongo: @@ -23,7 +21,9 @@ services: POSTGRES_PASSWORD: ${PG_PASSWORD:-postgres} POSTGRES_DB: ${PG_DATABASE:-commonly} volumes: - - postgres-data-dev:/var/lib/postgresql/data + # Local dev uses postgres:latest; mounting the parent dir keeps pace with + # versioned PGDATA paths like /var/lib/postgresql//docker. + - postgres-data-dev:/var/lib/postgresql networks: - app-network-dev @@ -50,7 +50,6 @@ services: - PG_HOST=${PG_HOST:-postgres} - PG_PORT=${PG_PORT:-5432} - PG_DATABASE=${PG_DATABASE:-commonly} - - PG_SSL_CA_PATH=/app/ca.pem # Authentication - JWT_SECRET=${JWT_SECRET:-development-jwt-secret} @@ -119,8 +118,6 @@ services: - ./:/repo:ro # Prevent overwriting node_modules - /app/node_modules - # Mount CA certificate - - ./ca.pem:/app/ca.pem:ro networks: - app-network-dev # Development: restart on crashes @@ -288,6 +285,7 @@ services: - COMMONLY_USER_TOKEN=${COMMONLY_SUMMARIZER_USER_TOKEN} - COMMONLY_AGENT_POLL_MS=${COMMONLY_AGENT_POLL_MS:-5000} - COMMONLY_AGENT_CONFIG_PATH=/app/external/commonly-bot-state/runtime.json + - COMMONLY_AGENT_OPTIONAL=1 volumes: - ./external:/app/external:ro - ./external/commonly-bot-state:/app/external/commonly-bot-state:ro diff --git a/docs/SELF_HOSTING.md b/docs/SELF_HOSTING.md index 0751fa6e..95e1cce3 100644 --- a/docs/SELF_HOSTING.md +++ b/docs/SELF_HOSTING.md @@ -7,14 +7,14 @@ It is intended for local self-hosting and evaluation. It uses the repository's e ## What This Uses - `docker-compose.dev.yml` for the default local development stack -- `docker-compose.yml` for the base stack used by the backend for local runtime orchestration +- `docker-compose.yml` for the base/production-style stack and backend runtime orchestration defaults - `./dev.sh` as the simplest wrapper for common local lifecycle commands ## Prerequisites - Docker Engine with the Compose plugin or `docker-compose` - Git -- A local `.env` file in the repository root +- A local `.env` file in the repository root (`./dev.sh up` will create one from `.env.example` if missing) Optional: @@ -26,9 +26,10 @@ Optional: ```bash git clone https://github.com/Team-Commonly/commonly.git cd commonly +cp .env.example .env ``` -Create a root `.env` file with at least the values needed for local startup: +The checked-in `.env.example` already contains safe local defaults. The smallest required values are: ```dotenv NODE_ENV=development @@ -42,13 +43,12 @@ PG_PASSWORD=postgres PG_HOST=postgres PG_PORT=5432 PG_DATABASE=commonly -PG_SSL_CA_PATH=/app/ca.pem REACT_APP_API_URL=http://localhost:5000 FRONTEND_URL=http://localhost:3000 ``` -If you are using an external PostgreSQL instance that requires a CA certificate, fetch it before starting the stack: +For the default Docker Compose Postgres container, leave SSL disabled. Only fetch `ca.pem` if you are pointing Commonly at an external PostgreSQL instance that requires a CA certificate: ```bash node download-ca.js @@ -85,6 +85,7 @@ docker-compose -f docker-compose.dev.yml build - LiteLLM: start with `docker-compose -f docker-compose.dev.yml --profile litellm up -d` - Clawdbot services: start with `./dev.sh clawdbot up` +- AI-backed features: set `GEMINI_API_KEY`, `OPENAI_API_KEY`, or `OPENROUTER_API_KEY` in `.env`, then run `./dev.sh restart` Leave related environment variables unset unless you are actively enabling those services. @@ -93,7 +94,9 @@ Leave related environment variables unset unless you are actively enabling those - Keep secrets in `.env` and out of version control. - The first boot can take longer because the development containers may install dependencies on startup. - `docker-compose.dev.yml` is the right default for local self-hosting on this branch. -- `docker-compose.yml` is still used by backend runtime orchestration settings, so avoid renaming or moving it without updating those environment variables. +- `docker-compose.dev.yml` contains local-only fixes such as the Postgres volume target used by `./dev.sh`. +- `docker-compose.yml` intentionally preserves the original base/production-style behavior, so local Docker fixes should stay scoped to `docker-compose.dev.yml` unless you have verified they are safe for non-local environments. +- The local `commonly-bot` container no longer needs a runtime token before the first boot. It waits idle until you provision it from Agents Hub. ## Rollback diff --git a/docs/agents/AGENT_RUNTIME.md b/docs/agents/AGENT_RUNTIME.md index 3fca2c10..cdc3ca26 100644 --- a/docs/agents/AGENT_RUNTIME.md +++ b/docs/agents/AGENT_RUNTIME.md @@ -216,11 +216,14 @@ Skill credential overrides: ## Docker Compose (dev) -`docker-compose.dev.yml` includes a `commonly-bot` service. It requires a runtime token for `commonly-bot`: +`docker-compose.dev.yml` includes a `commonly-bot` service. In local dev it can boot without a runtime token and will wait idle until one is provisioned. To activate it: 1. Install Commonly Bot in Agent Hub for the target pod. 2. Issue a runtime token from the agent config dialog. -3. Set `COMMONLY_SUMMARIZER_RUNTIME_TOKEN` before `./dev.sh up` (or restart the service). +3. Either set `COMMONLY_SUMMARIZER_RUNTIME_TOKEN` before `./dev.sh up`, or provision `commonly-bot` from Agents Hub later and restart the service if needed. + +Local-dev note: +- `./dev.sh up` creates `.env` from `.env.example` if missing and prints guidance when optional AI provider keys are unset. Defaults: - `COMMONLY_BASE_URL=http://backend:5000` diff --git a/docs/deployment/DEPLOYMENT.md b/docs/deployment/DEPLOYMENT.md index 6b6132e2..123903ce 100644 --- a/docs/deployment/DEPLOYMENT.md +++ b/docs/deployment/DEPLOYMENT.md @@ -56,7 +56,6 @@ PG_PASSWORD=postgres PG_HOST=postgres PG_PORT=5432 PG_DATABASE=commonly -PG_SSL_CA_PATH=/app/ca.pem # Frontend REACT_APP_API_URL=http://localhost:5000 @@ -104,7 +103,12 @@ CLAWDBOT_GATEWAY_TOKEN=dev-token CLAWDBOT_GATEWAY_URL=http://clawdbot-gateway:18789 ``` -2. Download the CA certificate (if using external PostgreSQL): +Notes: +- For the default Docker Compose Postgres container, leave SSL disabled. `PG_SSL_CA_PATH` is only for external PostgreSQL instances that require a CA certificate. +- The repository also ships `.env.example` with these local defaults; `./dev.sh up` will create `.env` from it automatically if missing. +- AI keys are optional for boot. If `GEMINI_API_KEY`, `OPENAI_API_KEY`, and `OPENROUTER_API_KEY` are all unset, the app still starts but AI-backed features will stay limited until you add a key and restart. + +2. Download the CA certificate only if you are using external PostgreSQL: ```bash node download-ca.js diff --git a/external/commonly-agent-services/commonly-bot/index.js b/external/commonly-agent-services/commonly-bot/index.js index c481ba58..527abd25 100644 --- a/external/commonly-agent-services/commonly-bot/index.js +++ b/external/commonly-agent-services/commonly-bot/index.js @@ -4,6 +4,7 @@ const baseUrl = process.env.COMMONLY_BASE_URL || 'http://localhost:5000'; const token = process.env.COMMONLY_AGENT_TOKEN; const userToken = process.env.COMMONLY_USER_TOKEN; const configPath = process.env.COMMONLY_AGENT_CONFIG_PATH; +const optionalRuntime = process.env.COMMONLY_AGENT_OPTIONAL === '1'; const SOCIAL_POST_LIMIT = parseInt(process.env.COMMONLY_SOCIAL_CURATION_LIMIT, 10) || 60; const ENABLE_SOCIAL_REPHRASE = process.env.COMMONLY_SOCIAL_REPHRASE_ENABLED !== '0'; const ENABLE_SOCIAL_FEED_POST = process.env.COMMONLY_SOCIAL_POST_TO_FEED === '1'; @@ -61,6 +62,20 @@ const buildUserHeaders = (tokenValue) => ({ 'Content-Type': 'application/json', }); +let waitingForAccounts = false; + +const logWaitingForAccounts = () => { + if (waitingForAccounts) return; + waitingForAccounts = true; + console.warn('No agent tokens configured yet. Waiting for COMMONLY_AGENT_TOKEN or COMMONLY_AGENT_CONFIG_PATH.'); +}; + +const logAccountsDetected = (count) => { + if (!waitingForAccounts) return; + waitingForAccounts = false; + console.log(`Runtime config detected. Resuming polling for ${count} account(s).`); +}; + const formatIntegrationSummary = (summary, sourceOverride) => { if (!summary) return ''; const source = sourceOverride || summary.source || 'external'; @@ -810,27 +825,44 @@ if (userToken) { const initialAccounts = buildAccounts(); if (initialAccounts.length === 0) { - console.error('No agent tokens configured. Set COMMONLY_AGENT_TOKEN or COMMONLY_AGENT_CONFIG_PATH.'); - process.exit(1); + if (optionalRuntime) { + logWaitingForAccounts(); + } else { + console.error('No agent tokens configured. Set COMMONLY_AGENT_TOKEN or COMMONLY_AGENT_CONFIG_PATH.'); + process.exit(1); + } } -Promise.all( - initialAccounts.map((account) => ( - fetchEvents(account.runtimeToken) - .then((events) => { - console.log(`Commonly Bot connected (${account.id}). ${events.length} pending events.`); - }) - .catch((err) => { - console.error(`Commonly Bot connection failed (${account.id}):`, err.message); - }) - )), -).catch(() => {}); +if (initialAccounts.length > 0) { + Promise.all( + initialAccounts.map((account) => ( + fetchEvents(account.runtimeToken) + .then((events) => { + console.log(`Commonly Bot connected (${account.id}). ${events.length} pending events.`); + }) + .catch((err) => { + console.error(`Commonly Bot connection failed (${account.id}):`, err.message); + }) + )), + ).catch(() => {}); +} let isPolling = false; setInterval(async () => { if (isPolling) return; isPolling = true; const accounts = buildAccounts(); + if (accounts.length === 0) { + if (optionalRuntime) { + logWaitingForAccounts(); + isPolling = false; + return; + } + console.error('No agent tokens configured. Set COMMONLY_AGENT_TOKEN or COMMONLY_AGENT_CONFIG_PATH.'); + isPolling = false; + return; + } + logAccountsDetected(accounts.length); for (const account of accounts) { // eslint-disable-next-line no-await-in-loop await pollAccount(account); From def3be366391f3c5b0f17de570d3c16159802c11 Mon Sep 17 00:00:00 2001 From: Mubai Hua Date: Sun, 26 Apr 2026 14:07:54 -0700 Subject: [PATCH 2/2] Address local dev bootstrap review notes --- dev.sh | 17 +++++------------ docker-compose.dev.yml | 4 ++-- docs/agents/AGENT_RUNTIME.md | 2 +- .../commonly-bot/index.js | 3 +-- 4 files changed, 9 insertions(+), 17 deletions(-) diff --git a/dev.sh b/dev.sh index f8e030fe..49b89430 100755 --- a/dev.sh +++ b/dev.sh @@ -17,24 +17,17 @@ HELM_CHART="k8s/helm/commonly" ENV_FILE=".env" ENV_EXAMPLE_FILE=".env.example" -get_env_value() { +has_nonempty_env_value() { local key="$1" if [ -n "${!key:-}" ]; then - printf '%s' "${!key}" - return + return 0 fi - if [ -f "$ENV_FILE" ]; then - local line - line=$(grep -E "^${key}=" "$ENV_FILE" | tail -n 1 || true) - if [ -n "$line" ]; then - printf '%s' "${line#*=}" - fi + if [ ! -f "$ENV_FILE" ]; then + return 1 fi -} -has_nonempty_env_value() { - [ -n "$(get_env_value "$1")" ] + grep -Eq "^${key}=.+" "$ENV_FILE" } ensure_local_env_file() { diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 2d5716d9..082cb365 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -21,8 +21,8 @@ services: POSTGRES_PASSWORD: ${PG_PASSWORD:-postgres} POSTGRES_DB: ${PG_DATABASE:-commonly} volumes: - # Local dev uses postgres:latest; mounting the parent dir keeps pace with - # versioned PGDATA paths like /var/lib/postgresql//docker. + # postgres:latest can set PGDATA to a major-versioned path such as + # /var/lib/postgresql/18/docker, so local dev mounts the parent dir. - postgres-data-dev:/var/lib/postgresql networks: - app-network-dev diff --git a/docs/agents/AGENT_RUNTIME.md b/docs/agents/AGENT_RUNTIME.md index cdc3ca26..0e701a97 100644 --- a/docs/agents/AGENT_RUNTIME.md +++ b/docs/agents/AGENT_RUNTIME.md @@ -220,7 +220,7 @@ Skill credential overrides: 1. Install Commonly Bot in Agent Hub for the target pod. 2. Issue a runtime token from the agent config dialog. -3. Either set `COMMONLY_SUMMARIZER_RUNTIME_TOKEN` before `./dev.sh up`, or provision `commonly-bot` from Agents Hub later and restart the service if needed. +3. Either set `COMMONLY_SUMMARIZER_RUNTIME_TOKEN` before `./dev.sh up`, or provision `commonly-bot` from Agents Hub later; the local bot rereads the generated runtime config on its next poll. Local-dev note: - `./dev.sh up` creates `.env` from `.env.example` if missing and prints guidance when optional AI provider keys are unset. diff --git a/external/commonly-agent-services/commonly-bot/index.js b/external/commonly-agent-services/commonly-bot/index.js index 527abd25..980e1f47 100644 --- a/external/commonly-agent-services/commonly-bot/index.js +++ b/external/commonly-agent-services/commonly-bot/index.js @@ -859,8 +859,7 @@ setInterval(async () => { return; } console.error('No agent tokens configured. Set COMMONLY_AGENT_TOKEN or COMMONLY_AGENT_CONFIG_PATH.'); - isPolling = false; - return; + process.exit(1); } logAccountsDetected(accounts.length); for (const account of accounts) {