Improve local Docker dev bootstrap

dev.sh

@@ -14,9 +14,71 @@ CLUSTER_NAME="${COMMONLY_CLUSTER_NAME:-commonly-local}"
 K8S_NAMESPACE="${COMMONLY_K8S_NAMESPACE:-commonly-local}"
 HELM_RELEASE="${COMMONLY_HELM_RELEASE:-commonly}"
 HELM_CHART="k8s/helm/commonly"
+ENV_FILE=".env"
+ENV_EXAMPLE_FILE=".env.example"
+
+has_nonempty_env_value() {
+    local key="$1"
+    if [ -n "${!key:-}" ]; then
+        return 0
+    fi
+
+    if [ ! -f "$ENV_FILE" ]; then
+        return 1
+    fi
+
+    grep -Eq "^${key}=.+" "$ENV_FILE"
+}
+
+ensure_local_env_file() {
+    if [ -f "$ENV_FILE" ]; then
+        return
+    fi
+
+    if [ -f "$ENV_EXAMPLE_FILE" ]; then
+        cp "$ENV_EXAMPLE_FILE" "$ENV_FILE"
+        echo "📝 Created $ENV_FILE from $ENV_EXAMPLE_FILE"
+        echo "   Review it before enabling optional integrations or AI features."
+        echo ""
+        return
+    fi
+
+    echo "⚠️  $ENV_FILE is missing and $ENV_EXAMPLE_FILE was not found."
+    echo "   Docker Compose may still start if your shell env already has the needed values."
+    echo ""
+}
+
+print_local_dev_notes() {
+    echo ""
+    echo "Local dev notes:"
+
+    if has_nonempty_env_value "GEMINI_API_KEY" \
+        || has_nonempty_env_value "OPENAI_API_KEY" \
+        || has_nonempty_env_value "OPENROUTER_API_KEY"; then
+        echo "  • AI provider credentials detected."
+    else
+        echo "  • No AI provider key configured. AI features will stay limited until you set one of:"
+        echo "    GEMINI_API_KEY, OPENAI_API_KEY, or OPENROUTER_API_KEY in $ENV_FILE"
+        echo "    Then run: ./dev.sh restart"
+    fi
+
+    if has_nonempty_env_value "COMMONLY_SUMMARIZER_RUNTIME_TOKEN"; then
+        echo "  • commonly-bot runtime token detected."
+    else
+        echo "  • commonly-bot will stay idle until you provision it from Agents Hub."
+        echo "    That is expected for fresh local setups."
+    fi
+
+    if has_nonempty_env_value "CLAWDBOT_GATEWAY_TOKEN"; then
+        echo "  • Clawdbot gateway token detected. Start it with: ./dev.sh clawdbot up"
+    else
+        echo "  • Clawdbot services are optional. Add CLAWDBOT_GATEWAY_TOKEN in $ENV_FILE if you want local OpenClaw."
+    fi
+}
 
 case "$1" in
     up)
+        ensure_local_env_file
         echo "🚀 Starting development environment..."
         # Set higher timeout for frontend npm install
         export COMPOSE_HTTP_TIMEOUT=300
@@ -28,6 +90,7 @@ case "$1" in
         echo "🐘 PostgreSQL: localhost:5432"
         echo ""
         echo "💡 To start Clawdbot: ./dev.sh clawdbot up"
+        print_local_dev_notes
         ;;
 
     down)
@@ -37,9 +100,11 @@ case "$1" in
         ;;
 
     restart)
+        ensure_local_env_file
         echo "🔄 Restarting development environment..."
         docker-compose -f $COMPOSE_FILE restart
         echo "✅ Development environment restarted!"
+        print_local_dev_notes
         ;;
 
     logs)
@@ -295,4 +360,4 @@ case "$1" in
         echo "  ./dev.sh cluster test"
         echo "  ./dev.sh cluster down"
         ;;
-esac
+esac

docker-compose.dev.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   # MongoDB service
   mongo:
@@ -23,7 +21,9 @@ services:
       POSTGRES_PASSWORD: ${PG_PASSWORD:-postgres}
       POSTGRES_DB: ${PG_DATABASE:-commonly}
     volumes:
-      - postgres-data-dev:/var/lib/postgresql/data
+      # postgres:latest can set PGDATA to a major-versioned path such as
+      # /var/lib/postgresql/18/docker, so local dev mounts the parent dir.
+      - postgres-data-dev:/var/lib/postgresql
     networks:
       - app-network-dev
 
@@ -50,7 +50,6 @@ services:
       - PG_HOST=${PG_HOST:-postgres}
       - PG_PORT=${PG_PORT:-5432}
       - PG_DATABASE=${PG_DATABASE:-commonly}
-      - PG_SSL_CA_PATH=/app/ca.pem
 
       # Authentication
       - JWT_SECRET=${JWT_SECRET:-development-jwt-secret}
@@ -119,8 +118,6 @@ services:
       - ./:/repo:ro
       # Prevent overwriting node_modules
       - /app/node_modules
-      # Mount CA certificate
-      - ./ca.pem:/app/ca.pem:ro
     networks:
       - app-network-dev
     # Development: restart on crashes
@@ -288,6 +285,7 @@ services:
       - COMMONLY_USER_TOKEN=${COMMONLY_SUMMARIZER_USER_TOKEN}
       - COMMONLY_AGENT_POLL_MS=${COMMONLY_AGENT_POLL_MS:-5000}
       - COMMONLY_AGENT_CONFIG_PATH=/app/external/commonly-bot-state/runtime.json
+      - COMMONLY_AGENT_OPTIONAL=1
     volumes:
       - ./external:/app/external:ro
       - ./external/commonly-bot-state:/app/external/commonly-bot-state:ro

docs/SELF_HOSTING.md

@@ -7,14 +7,14 @@ It is intended for local self-hosting and evaluation. It uses the repository's e
 ## What This Uses
 
 - `docker-compose.dev.yml` for the default local development stack
-- `docker-compose.yml` for the base stack used by the backend for local runtime orchestration
+- `docker-compose.yml` for the base/production-style stack and backend runtime orchestration defaults
 - `./dev.sh` as the simplest wrapper for common local lifecycle commands
 
 ## Prerequisites
 
 - Docker Engine with the Compose plugin or `docker-compose`
 - Git
-- A local `.env` file in the repository root
+- A local `.env` file in the repository root (`./dev.sh up` will create one from `.env.example` if missing)
 
 Optional:
 
@@ -26,9 +26,10 @@ Optional:
 ```bash
 git clone https://github.com/Team-Commonly/commonly.git
 cd commonly
+cp .env.example .env
 ```
 
-Create a root `.env` file with at least the values needed for local startup:
+The checked-in `.env.example` already contains safe local defaults. The smallest required values are:
 
 ```dotenv
 NODE_ENV=development
@@ -42,13 +43,12 @@ PG_PASSWORD=postgres
 PG_HOST=postgres
 PG_PORT=5432
 PG_DATABASE=commonly
-PG_SSL_CA_PATH=/app/ca.pem
 
 REACT_APP_API_URL=http://localhost:5000
 FRONTEND_URL=http://localhost:3000
 ```
 
-If you are using an external PostgreSQL instance that requires a CA certificate, fetch it before starting the stack:
+For the default Docker Compose Postgres container, leave SSL disabled. Only fetch `ca.pem` if you are pointing Commonly at an external PostgreSQL instance that requires a CA certificate:
 
 ```bash
 node download-ca.js
@@ -85,6 +85,7 @@ docker-compose -f docker-compose.dev.yml build
 
 - LiteLLM: start with `docker-compose -f docker-compose.dev.yml --profile litellm up -d`
 - Clawdbot services: start with `./dev.sh clawdbot up`
+- AI-backed features: set `GEMINI_API_KEY`, `OPENAI_API_KEY`, or `OPENROUTER_API_KEY` in `.env`, then run `./dev.sh restart`
 
 Leave related environment variables unset unless you are actively enabling those services.
 
@@ -93,7 +94,9 @@ Leave related environment variables unset unless you are actively enabling those
 - Keep secrets in `.env` and out of version control.
 - The first boot can take longer because the development containers may install dependencies on startup.
 - `docker-compose.dev.yml` is the right default for local self-hosting on this branch.
-- `docker-compose.yml` is still used by backend runtime orchestration settings, so avoid renaming or moving it without updating those environment variables.
+- `docker-compose.dev.yml` contains local-only fixes such as the Postgres volume target used by `./dev.sh`.
+- `docker-compose.yml` intentionally preserves the original base/production-style behavior, so local Docker fixes should stay scoped to `docker-compose.dev.yml` unless you have verified they are safe for non-local environments.
+- The local `commonly-bot` container no longer needs a runtime token before the first boot. It waits idle until you provision it from Agents Hub.
 
 ## Rollback
 

docs/agents/AGENT_RUNTIME.md

@@ -216,11 +216,14 @@ Skill credential overrides:
 
 ## Docker Compose (dev)
 
-`docker-compose.dev.yml` includes a `commonly-bot` service. It requires a runtime token for `commonly-bot`:
+`docker-compose.dev.yml` includes a `commonly-bot` service. In local dev it can boot without a runtime token and will wait idle until one is provisioned. To activate it:
 
 1. Install Commonly Bot in Agent Hub for the target pod.
 2. Issue a runtime token from the agent config dialog.
-3. Set `COMMONLY_SUMMARIZER_RUNTIME_TOKEN` before `./dev.sh up` (or restart the service).
+3. Either set `COMMONLY_SUMMARIZER_RUNTIME_TOKEN` before `./dev.sh up`, or provision `commonly-bot` from Agents Hub later; the local bot rereads the generated runtime config on its next poll.
+
+Local-dev note:
+- `./dev.sh up` creates `.env` from `.env.example` if missing and prints guidance when optional AI provider keys are unset.
 
 Defaults:
 - `COMMONLY_BASE_URL=http://backend:5000`

docs/deployment/DEPLOYMENT.md

@@ -56,7 +56,6 @@ PG_PASSWORD=postgres
 PG_HOST=postgres
 PG_PORT=5432
 PG_DATABASE=commonly
-PG_SSL_CA_PATH=/app/ca.pem
 
 # Frontend
 REACT_APP_API_URL=http://localhost:5000
@@ -104,7 +103,12 @@ CLAWDBOT_GATEWAY_TOKEN=dev-token
 CLAWDBOT_GATEWAY_URL=http://clawdbot-gateway:18789
 ```
 
-2. Download the CA certificate (if using external PostgreSQL):
+Notes:
+- For the default Docker Compose Postgres container, leave SSL disabled. `PG_SSL_CA_PATH` is only for external PostgreSQL instances that require a CA certificate.
+- The repository also ships `.env.example` with these local defaults; `./dev.sh up` will create `.env` from it automatically if missing.
+- AI keys are optional for boot. If `GEMINI_API_KEY`, `OPENAI_API_KEY`, and `OPENROUTER_API_KEY` are all unset, the app still starts but AI-backed features will stay limited until you add a key and restart.
+
+2. Download the CA certificate only if you are using external PostgreSQL:
 
 ```bash
 node download-ca.js

external/commonly-agent-services/commonly-bot/index.js

@@ -4,6 +4,7 @@ const baseUrl = process.env.COMMONLY_BASE_URL || 'http://localhost:5000';
 const token = process.env.COMMONLY_AGENT_TOKEN;
 const userToken = process.env.COMMONLY_USER_TOKEN;
 const configPath = process.env.COMMONLY_AGENT_CONFIG_PATH;
+const optionalRuntime = process.env.COMMONLY_AGENT_OPTIONAL === '1';
 const SOCIAL_POST_LIMIT = parseInt(process.env.COMMONLY_SOCIAL_CURATION_LIMIT, 10) || 60;
 const ENABLE_SOCIAL_REPHRASE = process.env.COMMONLY_SOCIAL_REPHRASE_ENABLED !== '0';
 const ENABLE_SOCIAL_FEED_POST = process.env.COMMONLY_SOCIAL_POST_TO_FEED === '1';
@@ -61,6 +62,20 @@ const buildUserHeaders = (tokenValue) => ({
   'Content-Type': 'application/json',
 });
 
+let waitingForAccounts = false;
+
+const logWaitingForAccounts = () => {
+  if (waitingForAccounts) return;
+  waitingForAccounts = true;
+  console.warn('No agent tokens configured yet. Waiting for COMMONLY_AGENT_TOKEN or COMMONLY_AGENT_CONFIG_PATH.');
+};
+
+const logAccountsDetected = (count) => {
+  if (!waitingForAccounts) return;
+  waitingForAccounts = false;
+  console.log(`Runtime config detected. Resuming polling for ${count} account(s).`);
+};
+
 const formatIntegrationSummary = (summary, sourceOverride) => {
   if (!summary) return '';
   const source = sourceOverride || summary.source || 'external';
@@ -810,27 +825,43 @@ if (userToken) {
 
 const initialAccounts = buildAccounts();
 if (initialAccounts.length === 0) {
-  console.error('No agent tokens configured. Set COMMONLY_AGENT_TOKEN or COMMONLY_AGENT_CONFIG_PATH.');
-  process.exit(1);
+  if (optionalRuntime) {
+    logWaitingForAccounts();
+  } else {
+    console.error('No agent tokens configured. Set COMMONLY_AGENT_TOKEN or COMMONLY_AGENT_CONFIG_PATH.');
+    process.exit(1);
+  }
 }
 
-Promise.all(
-  initialAccounts.map((account) => (
-    fetchEvents(account.runtimeToken)
-      .then((events) => {
-        console.log(`Commonly Bot connected (${account.id}). ${events.length} pending events.`);
-      })
-      .catch((err) => {
-        console.error(`Commonly Bot connection failed (${account.id}):`, err.message);
-      })
-  )),
-).catch(() => {});
+if (initialAccounts.length > 0) {
+  Promise.all(
+    initialAccounts.map((account) => (
+      fetchEvents(account.runtimeToken)
+        .then((events) => {
+          console.log(`Commonly Bot connected (${account.id}). ${events.length} pending events.`);
+        })
+        .catch((err) => {
+          console.error(`Commonly Bot connection failed (${account.id}):`, err.message);
+        })
+    )),
+  ).catch(() => {});
+}
 
 let isPolling = false;
 setInterval(async () => {
   if (isPolling) return;
   isPolling = true;
   const accounts = buildAccounts();
+  if (accounts.length === 0) {
+    if (optionalRuntime) {
+      logWaitingForAccounts();
+      isPolling = false;
+      return;
+    }
+    console.error('No agent tokens configured. Set COMMONLY_AGENT_TOKEN or COMMONLY_AGENT_CONFIG_PATH.');
+    process.exit(1);
+  }
+  logAccountsDetected(accounts.length);
   for (const account of accounts) {
     // eslint-disable-next-line no-await-in-loop
     await pollAccount(account);