From 1694cc717faba1528f048643c5c02cd1e3aa2d23 Mon Sep 17 00:00:00 2001 From: wjk <1471511095@qq.com> Date: Thu, 23 Jul 2026 19:45:43 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat(=E9=83=A8=E7=BD=B2):=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=20Ubuntu/Debian=20=E5=8E=9F=E7=94=9F=E4=B8=80?= =?UTF-8?q?=E9=94=AE=E9=83=A8=E7=BD=B2=EF=BC=88DataLink=20=E5=A4=96?= =?UTF-8?q?=E7=BD=AE=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 提供 deploy.sh 原生部署链路（configure/install/build/start/stop/doctor/tui）， 仅托管 Web + API；补齐无内置 DataLink 的 npm start / stack-runner。 语义 Data Link 改为外置 MCP 对接，deploy 不安装、不启动、不作健康门槛。 基于 main reset 后重提原 #82。 --- .env.example | 3 + .github/workflows/ci.yml | 31 ++ README.md | 44 +- README_zh.md | 41 +- apps/web/.env.example | 4 + deploy.sh | 16 + docs/en/index.md | 2 +- docs/en/quick-start.md | 149 +++-- docs/zh/index.md | 2 +- docs/zh/quick-start.md | 149 +++-- package.json | 3 + scripts/deploy/args.mjs | 87 +++ scripts/deploy/args.test.mjs | 82 +++ scripts/deploy/bootstrap.sh | 144 +++++ scripts/deploy/bootstrap.test.mjs | 163 ++++++ scripts/deploy/cli.mjs | 719 +++++++++++++++++++++++++ scripts/deploy/cli.test.mjs | 372 +++++++++++++ scripts/deploy/config.mjs | 218 ++++++++ scripts/deploy/config.test.mjs | 130 +++++ scripts/deploy/controller.mjs | 157 ++++++ scripts/deploy/controller.test.mjs | 170 ++++++ scripts/deploy/dependencies.mjs | 133 +++++ scripts/deploy/dependencies.test.mjs | 88 +++ scripts/deploy/health.mjs | 92 ++++ scripts/deploy/health.test.mjs | 174 ++++++ scripts/deploy/install-dependency.sh | 46 ++ scripts/deploy/lifecycle.test.mjs | 335 ++++++++++++ scripts/deploy/ports.mjs | 127 +++++ scripts/deploy/ports.test.mjs | 136 +++++ scripts/deploy/process-state.mjs | 241 +++++++++ scripts/deploy/process-state.test.mjs | 156 ++++++ scripts/deploy/smoke-native-deploy.mjs | 223 ++++++++ scripts/stack-runner.mjs | 122 +++++ scripts/stack-runtime-config.mjs | 27 + scripts/stack-runtime-config.test.mjs | 31 ++ scripts/start.mjs | 4 + 36 files changed, 4542 insertions(+), 79 deletions(-) create mode 100755 deploy.sh create mode 100644 scripts/deploy/args.mjs create mode 100644 scripts/deploy/args.test.mjs create mode 100755 scripts/deploy/bootstrap.sh create mode 100644 scripts/deploy/bootstrap.test.mjs create mode 100644 scripts/deploy/cli.mjs create mode 100644 scripts/deploy/cli.test.mjs create mode 100644 scripts/deploy/config.mjs create mode 100644 scripts/deploy/config.test.mjs create mode 100644 scripts/deploy/controller.mjs create mode 100644 scripts/deploy/controller.test.mjs create mode 100644 scripts/deploy/dependencies.mjs create mode 100644 scripts/deploy/dependencies.test.mjs create mode 100644 scripts/deploy/health.mjs create mode 100644 scripts/deploy/health.test.mjs create mode 100755 scripts/deploy/install-dependency.sh create mode 100644 scripts/deploy/lifecycle.test.mjs create mode 100644 scripts/deploy/ports.mjs create mode 100644 scripts/deploy/ports.test.mjs create mode 100644 scripts/deploy/process-state.mjs create mode 100644 scripts/deploy/process-state.test.mjs create mode 100644 scripts/deploy/smoke-native-deploy.mjs create mode 100644 scripts/stack-runner.mjs create mode 100644 scripts/stack-runtime-config.mjs create mode 100644 scripts/stack-runtime-config.test.mjs create mode 100644 scripts/start.mjs diff --git a/.env.example b/.env.example index 593c7577..5a652878 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,9 @@ API_HOST=127.0.0.1 API_PORT=8787 +WEB_HOST=0.0.0.0 +WEB_PORT=3000 +# LLM_* values are optional server-side defaults. Models can also be created in the Web UI after deploy. LLM_PROVIDER=openai-compatible LLM_MODEL=qwen-plus LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9da75ed2..2a1f2494 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,6 +97,37 @@ jobs: - name: Run built-in DTC Growth regression run: node --test scripts/test-builtin-dtc-growth-datasource.mjs + native-deploy-smoke: + name: Native Deploy Smoke + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version: "22" + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Run deployment unit tests + run: npm run test:deploy + + - name: Run native deployment smoke + run: npm run smoke:native-deploy + + - name: Upload redacted deployment logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: native-deploy-logs + path: storage/logs/ + if-no-files-found: ignore + docs: name: Docs runs-on: ubuntu-latest diff --git a/README.md b/README.md index 3cd32dd4..3da660a1 100644 --- a/README.md +++ b/README.md @@ -71,14 +71,40 @@ See the [v0.2.0 release notes](docs/en/releases/v0.2.0.md) for the complete capa ## 🚀 Formal deploy -Default path is the **formal** stack: `password` auth + `build` / `start` (do **not** run `npm run dev`). No business database required — built-in demo data sources, including the DTC Growth Review case, work out of the box. +Formal mode has two paths (do **not** run `npm run dev`). Docker / Compose is not provided in this release. -Two formal environments share the **same startup commands**: +### Recommended: Ubuntu / Debian one-click -| Environment | Use for | Email | Public URL | -| --- | --- | --- | --- | -| **Formal test** | Local / private acceptance | `AUTH_EMAIL_DELIVERY=test` (links in API console) | `http://127.0.0.1:3000` | -| **Real production** | Public service | `AUTH_EMAIL_DELIVERY=smtp` | Public HTTPS + reverse proxy | +`./deploy.sh` generates configuration, installs dependencies, builds (Web, API, and TUI), and starts Web + API as a detached background process — closing the terminal does not stop the stack. The TUI is built and ready but is a foreground client: start it in another terminal with `./deploy.sh tui` (or `npm run start:tui`); it does **not** stay running with the stack. Semantic Data Link is an **external** component (not started by deploy) — connect it later via MCP in the Web UI if needed. No model key is required during deploy — create and enable a model profile in the Web UI after login. Does **not** support native Windows / macOS. + +```bash +git clone https://github.com/datagallery-lab/datafoundry.git +cd datafoundry +./deploy.sh +``` + +If a complete `.env` already exists, interactive deploy skips the configuration questions. To change ports or the public URL (secrets are preserved; `.env` is backed up first): + +```bash +./deploy.sh deploy --reconfigure +``` + +Common management commands: + +```bash +./deploy.sh status +./deploy.sh start +./deploy.sh stop +./deploy.sh logs +./deploy.sh doctor +./deploy.sh tui # optional: foreground TUI client (API must already be up) +``` + +Open `http://127.0.0.1:3000/login`, register or sign in, create an OpenAI-compatible model profile, then go to `/data-tasks`. For remote hosts set `AUTH_PUBLIC_BASE_URL`. Re-running `./deploy.sh deploy` uses a maintenance window (stop the managed process group before install/build). + +### Windows / macOS / other: manual npm + +On native Windows, macOS, or other non-Ubuntu/Debian hosts, use manual npm. Install and run in the same environment; on Windows, do not share `node_modules` between Windows and WSL. ```bash git clone https://github.com/datagallery-lab/datafoundry.git @@ -116,8 +142,7 @@ Build and start: ```bash npm run build npm run build:web -npm run start:api # :8787 — /healthz liveness, /ready readiness -npm run start:web # :3000 — password mode via same-origin BFF +npm run start # Web :3000 + API :8787 ``` Open `http://127.0.0.1:3000/login`, register or sign in, go to `/data-tasks`, and ask: @@ -130,7 +155,8 @@ You will see the full chain: schema inspection → read-only SQL → SQL audit For real production, also configure SMTP and a reverse proxy: [`deploy/nginx.datafoundry.conf.example`](deploy/nginx.datafoundry.conf.example) (gzip/brotli static assets; keep `/api/copilotkit` uncompressed and unbuffered for SSE). -> Full steps and the two-environment matrix: [Quick Start](docs/en/quick-start.md). Contributor hot-reload (`npm run dev`) is appendix-only. Connect your own PostgreSQL / MySQL / CSV and more: [Data Sources guide](docs/en/guides/data-sources.md). +> Full steps, the two-environment matrix, and optional external Data Link notes: [Quick Start](docs/en/quick-start.md). Contributor hot-reload (`npm run dev`) is appendix-only. Connect your own PostgreSQL / MySQL / CSV and more: [Data Sources guide](docs/en/guides/data-sources.md). + ## 🆚 How It Differs From Coding Agents And SQL Chatbots diff --git a/README_zh.md b/README_zh.md index d56d7e96..eeabaed3 100644 --- a/README_zh.md +++ b/README_zh.md @@ -71,14 +71,40 @@ DataFoundry 0.2 在首个可用版本上，进一步补齐了有状态、可追 ## 🚀 正式态跑通 -默认按**正式态**部署：`password` 认证 + `build` / `start`（不要跑 `npm run dev`）。不需要准备业务数据库，包含 DTC 增长经营复盘在内的内置 demo 数据源开箱即用。 +正式态有两条路径（都不要跑 `npm run dev`）。本版本不提供 Docker / Compose。 -正式态分两种环境，**启动命令相同**： +### 推荐：Ubuntu / Debian 一键部署 -| 环境 | 用途 | 邮箱 | 公网地址 | -| --- | --- | --- | --- | -| **正式测试** | 本机 / 内网验收 | `AUTH_EMAIL_DELIVERY=test`（链接打控制台） | `http://127.0.0.1:3000` | -| **真实生产** | 对外服务 | `AUTH_EMAIL_DELIVERY=smtp` | 公网 HTTPS + 反代 | +`./deploy.sh` 自动生成配置、安装依赖、构建（含 Web、API 与 TUI），并以 detached 后台进程启动 Web + API——关闭终端一般不会停止服务。TUI 会在部署时构建就绪，但它是前台交互客户端：请另开终端执行 `./deploy.sh tui`（或 `npm run start:tui`），**不会**随 stack 后台常驻。语义 Data Link 为**外置**组件（deploy 不会安装或启动）——如需使用，稍后在 Web 的 MCP 配置中连接外部服务即可。部署时不要求填写模型 Key——登录后在 Web 中创建并启用模型即可。**不支持**原生 Windows / macOS。 + +```bash +git clone https://github.com/datagallery-lab/datafoundry.git +cd datafoundry +./deploy.sh +``` + +若已有完整 `.env`，交互部署会跳过配置问答。要改端口或公开访问地址时（会保留密钥并先备份 `.env`）： + +```bash +./deploy.sh deploy --reconfigure +``` + +常用管理命令： + +```bash +./deploy.sh status +./deploy.sh start +./deploy.sh stop +./deploy.sh logs +./deploy.sh doctor +./deploy.sh tui # 可选：前台启动 TUI（需 API 已在运行） +``` + +打开 `http://127.0.0.1:3000/login` 注册登录，在模型配置中创建 OpenAI-compatible Profile，然后进入 `/data-tasks`。远程部署请设置 `AUTH_PUBLIC_BASE_URL`；重复执行 `./deploy.sh deploy` 会进入维护窗口（先停止旧进程再安装/构建）。 + +### Windows / macOS / 其他：手动 npm + +原生 Windows、macOS 或其他非 Ubuntu/Debian 环境请用手动 npm。请在同一环境内安装和运行；Windows 用户不要在 Windows 和 WSL 之间共用 `node_modules`。 ```bash git clone https://github.com/datagallery-lab/datafoundry.git @@ -116,8 +142,7 @@ API_PROXY_TARGET=http://127.0.0.1:8787 ```bash npm run build npm run build:web -npm run start:api # :8787 — /healthz 存活，/ready 就绪 -npm run start:web # :3000 — password 模式走同源 BFF +npm run start # Web :3000 + API :8787 ``` 打开 `http://127.0.0.1:3000/login` 注册登录后进入 `/data-tasks`，提问： diff --git a/apps/web/.env.example b/apps/web/.env.example index 4299170a..190400a0 100755 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -1,5 +1,9 @@ # DataFoundry Web — formal deploy defaults (password + same-origin BFF). # +# Native one-click deploy (`./deploy.sh`) generates `apps/web/.env.local` for you. +# It uses same-origin BFF variables (empty NEXT_PUBLIC_* URLs) and sets +# API_PROXY_TARGET from the selected API host/port. +# # Keep in sync with root `.env` → DATAFOUNDRY_AUTH_MODE. # NEXT_PUBLIC_* values are baked in at `npm run build:web`. # diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 00000000..6be7eb35 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/deploy/bootstrap.sh +source "${ROOT_DIR}/scripts/deploy/bootstrap.sh" + +main() { + check_supported_system + ensure_node_22 "$@" + exec node "$ROOT_DIR/scripts/deploy/cli.mjs" "$@" +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/docs/en/index.md b/docs/en/index.md index 58845863..c25a869e 100644 --- a/docs/en/index.md +++ b/docs/en/index.md @@ -30,7 +30,7 @@ Review a realistic growth spike with schema inspection, read-only SQL, and trace ## Recommended path 1. Read [Product overview](overview.md) to confirm the problem space and capability boundaries. -2. Follow [Quick start](quick-start.md) to configure a model API key and run the built-in DTC Growth Review. +2. Follow [Quick start](quick-start.md): on Ubuntu / Debian prefer `./deploy.sh`; on Windows / macOS and other hosts use manual npm (`npm install` → configure → `build` / `start`). Then create a model profile in the Web UI and run the built-in DTC Growth Review. 3. Read [Capabilities](capabilities.md) to see coverage across Web, TUI, and backend API. 4. Choose [Web workbench guide](guides/web-workbench.md) or [TUI guide](guides/tui.md) based on your entry point. 5. When you need your own data, read [Data sources guide](guides/data-sources.md). diff --git a/docs/en/quick-start.md b/docs/en/quick-start.md index f838f361..f7291af4 100644 --- a/docs/en/quick-start.md +++ b/docs/en/quick-start.md @@ -1,8 +1,89 @@ # Quick start -This guide is for first-time DataFoundry deployers. After reading it, you can start the Web workbench in the **formal** stack (`build` + `start`, `password` auth), configure a model service, and run a data analysis task against the built-in DTC Growth Review data source. +This guide is for first-time DataFoundry deployers. Formal mode has two paths; **both** start the Web workbench with `password` auth (do **not** run `npm run dev`): -Formal mode has two environments. **Startup commands are the same**; the main differences are email delivery and the public base URL: +| Path | Hosts | Entry | +| --- | --- | --- | +| **Recommended: one-click** | Ubuntu / Debian | `./deploy.sh` (config, dependencies, build including TUI, detached Web/API start, and health checks in one flow) | +| **Manual npm** | Windows, macOS, other Linux, or hand-edited env files | `npm install` → configure `.env` → `npm run build` / `build:web` → `npm run start` | + +After deploy, configure a model in the Web UI and run an analysis against the built-in DTC Growth Review data source. Docker / Compose is **not** shipped in this release. + +## Requirements + +- **One-click deploy**: Ubuntu or Debian (x86_64 / aarch64); Node.js 22 (the script can help install it after consent) +- **Manual npm**: Linux, macOS, or Windows; Node.js >= 22 and npm +- Optional external [Data Link](https://github.com/datagallery-lab/datalink): run it as a separate process if you want semantic graph features (not required for deploy) + +Install and run the project in the same environment. On Windows, do not share `node_modules` between Windows and WSL. + +## Recommended: Ubuntu / Debian one-click deploy + +`./deploy.sh` does **not** support native Windows / macOS (use manual npm below). + +```bash +git clone https://github.com/datagallery-lab/datafoundry.git +cd datafoundry +./deploy.sh +``` + +On success Web + API keep running in the background (detached process group). Closing the terminal or pressing `Ctrl+C` in `./deploy.sh logs` does **not** stop DataFoundry — use `./deploy.sh stop`. One-click deploy also builds the TUI, but the TUI does **not** stay running with the stack; start it in another terminal when needed (see “Start the TUI” below). + +Open `http://127.0.0.1:3000/login` (or the Web URL printed by the script if the port differs), register and sign in, create/test/enable an OpenAI-compatible model profile, then go to `/data-tasks`. + +### Configuration rules + +- First run: the script generates `.env` and `apps/web/.env.local`, then confirms ports / public URL. No model key is required at deploy time. +- Later interactive `./deploy.sh` / `./deploy.sh deploy`: if a complete `.env` already exists, configuration questions are skipped. +- To change ports or the public URL again (existing secrets are kept; `.env` is backed up first): + +```bash +./deploy.sh deploy --reconfigure +``` + +- Unattended / CI defaults (no prompts; fails immediately on port conflicts or install that needs a sudo password): + +```bash +./deploy.sh deploy --non-interactive +``` + +`--reconfigure` and `--non-interactive` are mutually exclusive and only valid with `deploy`. + +### Lifecycle commands + +```bash +./deploy.sh status # process + API / Web health +./deploy.sh start # start an existing build (no install/build) +./deploy.sh stop # stop only the managed process group +./deploy.sh restart # stop then start (no install/build) +./deploy.sh logs # follow runtime logs; Ctrl+C does not stop the stack +./deploy.sh doctor # read-only dependency / config / port / disk / health checks +./deploy.sh tui # optional: foreground TUI client (API must be healthy; not a managed service) +./deploy.sh help +``` + +`LLM_*` is not required during deploy. Set `AUTH_PUBLIC_BASE_URL` for remote hosts. Re-running deploy uses a maintenance window: stop the managed process group before `npm ci` and builds. + +### External Data Link (optional) + +One-click deploy starts **only** Web + API. It does **not** install, start, or health-check Data Link. + +To use semantic features, run [Data Link](https://github.com/datagallery-lab/datalink) as a separate service (typically MCP on `:8080` and REST on `:8081`), then in the Web workbench MCP settings add an external server, for example: + +| Field | Example | +| --- | --- | +| `serverUrl` | `http://127.0.0.1:8080/mcp` | +| `apiUrl` | `http://127.0.0.1:8081` | +| `transport` | `streamable-http` | +| `toolManifest` | `[{ "name": "datalink_explore" }]` | + +Use a name/id containing `datalink` (or `datagraph`) so the Data Link panel can recognize it. + +## Windows / macOS / other: manual npm deploy + +`./deploy.sh` targets **Ubuntu / Debian only** and does not support native Windows / macOS. On Windows, macOS, or other distros, install, configure, and start with npm as below. Use the same path for hand-edited env files or split processes. Do **not** run `npm run dev` in formal environments. Contributor hot-reload is in the appendix. + +Formal environments: | Environment | Use for | `AUTH_EMAIL_DELIVERY` | `AUTH_PUBLIC_BASE_URL` | | --- | --- | --- | --- | @@ -11,18 +92,7 @@ Formal mode has two environments. **Startup commands are the same**; the main di Do **not** run `npm run dev` / `dev:api` / `dev:web` in either formal environment. Contributor hot-reload is in the appendix. -You do not need a business database for the first run. You only need Node.js, npm, and a model API key compatible with the OpenAI `/chat/completions` interface. - -## Requirements - -- Node.js >= 22 -- npm -- Linux, macOS, or Windows -- A model API key—for example Qwen, DeepSeek, or another OpenAI-compatible service - -On Windows, install and run the project in the same environment. Do not share `node_modules` between Windows and WSL. - -## 1. Install dependencies +### 1. Install dependencies From the repository root: @@ -33,18 +103,16 @@ npm install `node -v` must report 22 or higher. The first install generates the local DTC Growth Review SQLite fixture and compiles workspace dependencies; time depends on your machine and network. -## 2. Configure environment variables - -Copy the environment templates: +### 2. Configure environment variables ```bash cp .env.example .env cp apps/web/.env.example apps/web/.env.local ``` -### 2.1 Model (required for both formal environments) +#### 2.1 Model (optional server defaults) -Edit the root `.env` and set model configuration: +Edit the root `.env` for optional server-default models (you can also configure models only in the Web UI): ```bash LLM_PROVIDER=openai-compatible @@ -106,7 +174,7 @@ AUTH_SMTP_PASSWORD= Keep the frontend on `password`, empty public API URLs, and `API_PROXY_TARGET`. Put a reverse proxy in front; see [`deploy/nginx.datafoundry.conf.example`](https://github.com/datagallery-lab/datafoundry/blob/main/deploy/nginx.datafoundry.conf.example) — compress static assets; keep `/api/copilotkit` uncompressed and unbuffered for SSE. -## 3. Build and start (same for formal test and real production) +### 3. Build and start (same for formal test and real production) ```bash npm run build @@ -126,7 +194,7 @@ Open [http://127.0.0.1:3000/login](http://127.0.0.1:3000/login) (or your public After changing any `NEXT_PUBLIC_*` value in `apps/web/.env.local`, run `npm run build:web` again. -## 4. Run your first question +## Run your first question On `/data-tasks`: @@ -149,12 +217,19 @@ Compare GMV, gross margin, ad spend, and refunds by channel. Explain which chann When you see schema inspection, SQL execution, and result output, the path is working. -## 5. Start the TUI +## Start the TUI + +One-click deploy builds the TUI during the build stage, but does **not** auto-start it and does **not** treat it as a managed background service. With the API running, start the foreground client in another terminal: + +```bash +./deploy.sh tui +# or: npm run start:tui +``` -With the backend running: +Optionally point at the deployed API URL (defaults to `API_PORT` from `.env`): ```bash -npm run start:tui +./deploy.sh tui --runtime-url http://127.0.0.1:8787/api/copilotkit ``` Demo mode without a backend: @@ -171,7 +246,17 @@ npm run start:tui -- --resume More commands: [TUI guide](guides/tui.md). -## 6. Troubleshooting +## Troubleshooting + +For one-click deploy, start with: + +```bash +./deploy.sh status +./deploy.sh doctor +./deploy.sh logs +``` + +On the manual npm path, confirm `npm run start` is still running and check that terminal's output. ### Wrong Node version @@ -181,9 +266,11 @@ Fix: ```bash node -v +# one-click: +./deploy.sh doctor ``` -Upgrade to Node.js 22 or higher, then run `npm install` again. +Upgrade to Node.js 22 or higher, then retry. One-click deploy can also install Node after consent; on the manual path, re-run `npm install`. ### Page does not load @@ -191,9 +278,9 @@ Symptom: Browser cannot open the workbench URL. Fix: -- Confirm `npm run start:web` is still running (do not use `dev` in formal mode). -- Check whether port 3000 is in use. -- If 3000 is taken, use the frontend port shown in terminal output. +- Run `./deploy.sh status` (or confirm `npm run start` is still running on the manual path). Do not use `dev` in formal mode. +- Check whether port 3000 is in use; if the deploy script chose another port, use the URL it printed. +- If the process is stopped: `./deploy.sh start`. ### Backend not running @@ -202,6 +289,7 @@ Symptom: Page loads but questions get no response, or the resource panel fails t Fix: ```bash +./deploy.sh status curl http://127.0.0.1:8787/healthz curl http://127.0.0.1:8787/ready ``` @@ -209,7 +297,8 @@ curl http://127.0.0.1:8787/ready If the health check fails: ```bash -npm run start:api +./deploy.sh start +# manual path: npm run start ``` ### No verification email diff --git a/docs/zh/index.md b/docs/zh/index.md index c39ce769..d539ec05 100644 --- a/docs/zh/index.md +++ b/docs/zh/index.md @@ -30,7 +30,7 @@ DataFoundry 是面向数据分析场景的 AI 工作台，把自然语言提问 ## 推荐体验路径 1. 阅读 [产品概览](overview.md)，确认 DataFoundry 解决的问题和能力边界。 -2. 按 [快速开始](quick-start.md) 配置模型 API Key，并使用内置 DTC Growth Review 跑通第一个问题。 +2. 按 [快速开始](quick-start.md)：Ubuntu / Debian 推荐 `./deploy.sh`；Windows / macOS 及其他环境用手动 npm（`npm install` → 配置 → `build` / `start`）。登录 Web 后创建模型 Profile，再用内置 DTC Growth Review 跑通第一个问题。 3. 查看 [能力全览](capabilities.md)，了解 Web 工作台、TUI 和后端 API 的能力覆盖。 4. 根据使用入口选择 [Web 工作台指南](guides/web-workbench.md) 或 [TUI 指南](guides/tui.md)。 5. 需要接入自有数据时，再阅读 [数据源指南](guides/data-sources.md)。 diff --git a/docs/zh/quick-start.md b/docs/zh/quick-start.md index 13d6f893..b693df57 100644 --- a/docs/zh/quick-start.md +++ b/docs/zh/quick-start.md @@ -1,8 +1,89 @@ # 快速开始 -这篇文档面向第一次部署 DataFoundry 的用户。读完后，你可以按**正式态**启动 Web 工作台（`build` + `start`，`password` 认证），配置模型服务，用内置 DTC Growth Review 数据源跑通一次数据分析任务。 +这篇文档面向第一次部署 DataFoundry 的用户。正式态有两条路径，**启动后都是** `password` 认证的 Web 工作台（不要跑 `npm run dev`）： -正式态分两种环境，**启动命令相同**，差别主要在邮箱与公网地址： +| 路径 | 适用环境 | 入口 | +| --- | --- | --- | +| **推荐：一键部署** | Ubuntu / Debian | `./deploy.sh`（配置、依赖、构建含 TUI、detached 后台启动 Web/API 与健康检查一次完成） | +| **手动 npm** | Windows、macOS、其他 Linux，或需要手改环境变量时 | `npm install` → 配置 `.env` → `npm run build` / `build:web` → `npm run start` | + +部署完成后在 Web 中配置模型，再用内置 DTC Growth Review 数据源跑通一次分析。本版本**不提供** Docker / Compose。 + +## 环境要求 + +- **一键部署**：Ubuntu 或 Debian（x86_64 / aarch64）；Node.js 22（缺失时脚本可在确认后协助安装） +- **手动 npm**：Linux、macOS 或 Windows；Node.js >= 22 与 npm +- 可选外置 [Data Link](https://github.com/datagallery-lab/datalink)：需要语义图谱能力时单独运行（一键部署不依赖） + +请在同一环境内安装和运行项目。Windows 用户不要在 Windows 和 WSL 之间共用 `node_modules`。 + +## 推荐：Ubuntu / Debian 一键部署 + +`./deploy.sh` **不支持**原生 Windows / macOS（请改用下文手动 npm）。 + +```bash +git clone https://github.com/datagallery-lab/datafoundry.git +cd datafoundry +./deploy.sh +``` + +部署成功后 Web + API 在后台常驻（独立进程组）。关闭终端，或在 `./deploy.sh logs` 中按 `Ctrl+C`，**都不会**停止 DataFoundry——停止请用 `./deploy.sh stop`。一键部署会一并构建 TUI，但 TUI **不会**随 stack 后台常驻；需要时另开终端按成功提示启动（见下文「启动 TUI」）。 + +打开 `http://127.0.0.1:3000/login`（若端口被改过，以脚本打印的 Web 地址为准），注册并登录，在模型配置中创建、测试并启用 OpenAI-compatible Profile，然后进入 `/data-tasks`。 + +### 交互与配置规则 + +- 首次部署：脚本生成 `.env` 与 `apps/web/.env.local`，并确认端口 / 公开访问地址。部署阶段不要求填写模型 Key。 +- 之后再执行交互式 `./deploy.sh` / `./deploy.sh deploy`：若已有完整 `.env`，会跳过配置问答。 +- 需要重新配置端口或公开访问地址时（保留现有密钥，并先备份 `.env`）： + +```bash +./deploy.sh deploy --reconfigure +``` + +- 无人值守 / CI 默认（不提问；端口冲突或需要 sudo 密码的安装会立即失败）： + +```bash +./deploy.sh deploy --non-interactive +``` + +`--reconfigure` 与 `--non-interactive` 互斥，且仅对 `deploy` 有效。 + +### 生命周期命令 + +```bash +./deploy.sh status # 进程与 API / Web 健康状态 +./deploy.sh start # 用已有构建启动（不安装、不构建） +./deploy.sh stop # 只停止受管进程组 +./deploy.sh restart # 停止后启动（不安装、不构建） +./deploy.sh logs # 跟随运行日志；Ctrl+C 不停止服务 +./deploy.sh doctor # 只读检查依赖 / 配置 / 端口 / 磁盘 / 健康 +./deploy.sh tui # 可选：前台启动 TUI（需 API 已健康；不是受管后台服务） +./deploy.sh help +``` + +部署时不要求填写 `LLM_*`。远程主机请设置 `AUTH_PUBLIC_BASE_URL`。重复部署会进入维护窗口：先停止受管进程组，再执行 `npm ci` 与构建。 + +### 外置 Data Link（可选） + +一键部署**只启动** Web + API，**不会**安装、启动或健康检查 Data Link。 + +若需要语义能力，请单独运行 [Data Link](https://github.com/datagallery-lab/datalink)（常见为 MCP `:8080` + REST `:8081`），再在 Web 工作台的 MCP 设置中添加外部服务，例如： + +| 字段 | 示例 | +| --- | --- | +| `serverUrl` | `http://127.0.0.1:8080/mcp` | +| `apiUrl` | `http://127.0.0.1:8081` | +| `transport` | `streamable-http` | +| `toolManifest` | `[{ "name": "datalink_explore" }]` | + +名称/id 包含 `datalink`（或 `datagraph`）即可被 Data Link 面板识别。 + +## Windows / macOS / 其他：手动 npm 部署 + +`./deploy.sh` **仅面向 Ubuntu / Debian**，不支持原生 Windows / macOS。在 Windows、macOS 或其他发行版上，请按下列步骤用 npm 安装、配置并启动。需要手改环境变量或拆分进程时，也可走这条路径。两种正式态都**不要跑** `npm run dev`。贡献者热更新见文末附录。 + +正式态对照： | 环境 | 用途 | `AUTH_EMAIL_DELIVERY` | `AUTH_PUBLIC_BASE_URL` | | --- | --- | --- | --- | @@ -11,18 +92,7 @@ 两种正式态都**不要跑** `npm run dev` / `dev:api` / `dev:web`。贡献者本地热更新见文末附录。 -首次体验不需要准备业务数据库。你只需要 Node.js、npm 和一个兼容 OpenAI `/chat/completions` 接口的模型 API Key。 - -## 环境要求 - -- Node.js >= 22 -- npm -- Linux、macOS 或 Windows -- 一个模型 API Key，例如通义千问、DeepSeek 或其他 OpenAI-compatible 服务 - -Windows 用户请在同一个系统内安装和运行项目。不要在 Windows 和 WSL 之间共用 `node_modules`。 - -## 1. 安装依赖 +### 1. 安装依赖 在仓库根目录执行： @@ -33,18 +103,16 @@ npm install `node -v` 输出需要不低于 22。首次安装会生成本地 DTC Growth Review SQLite fixture，并编译工作区依赖；耗时取决于机器和网络。 -## 2. 配置环境变量 - -复制环境变量模板： +### 2. 配置环境变量 ```bash cp .env.example .env cp apps/web/.env.example apps/web/.env.local ``` -### 2.1 模型（两种正式态都要） +#### 2.1 模型（可选 server-default） -打开根目录 `.env`，填写模型配置： +打开根目录 `.env`，可填写可选的服务端默认模型（也可仅在 Web 中配置）： ```bash LLM_PROVIDER=openai-compatible @@ -106,7 +174,7 @@ AUTH_SMTP_PASSWORD= 前端同样保持 `password` + 空公开 API URL + `API_PROXY_TARGET`。对外入口请用反代，样例见 [`deploy/nginx.datafoundry.conf.example`](https://github.com/datagallery-lab/datafoundry/blob/main/deploy/nginx.datafoundry.conf.example)：静态资源压缩，SSE 路径 `/api/copilotkit` 关闭 gzip 与 `proxy_buffering`。 -## 3. 构建并启动（正式测试 / 真实生产相同） +### 3. 构建并启动（正式测试 / 真实生产相同） ```bash npm run build @@ -126,7 +194,7 @@ curl http://127.0.0.1:8787/ready # Mastra / builtin 就绪（含 startup_ms 改过 `apps/web/.env.local` 中的 `NEXT_PUBLIC_*` 后，需要重新执行 `npm run build:web`。 -## 4. 跑通第一个问题 +## 跑通第一个问题 打开 `/data-tasks` 后： @@ -149,12 +217,19 @@ curl http://127.0.0.1:8787/ready # Mastra / builtin 就绪（含 startup_ms 你看到 schema 检查、SQL 执行和结果产出后，说明链路已经跑通。 -## 5. 启动 TUI +## 启动 TUI + +一键部署会在构建阶段准备好 TUI，但**不会**自动启动，也不会把它当作后台受管进程。后端（API）运行后，另开终端启动前台客户端： + +```bash +./deploy.sh tui +# 或：npm run start:tui +``` -后端运行后，可以启动终端界面： +指定当前部署的 API 地址（可选；默认使用 `.env` 中的 `API_PORT`）： ```bash -npm run start:tui +./deploy.sh tui --runtime-url http://127.0.0.1:8787/api/copilotkit ``` 演示模式不需要后端： @@ -171,7 +246,17 @@ npm run start:tui -- --resume 更多命令见 [TUI 指南](guides/tui.md)。 -## 6. 排查 +## 排查 + +一键部署路径请先看： + +```bash +./deploy.sh status +./deploy.sh doctor +./deploy.sh logs +``` + +手动 npm 路径则确认 `npm run start` 仍在运行，并检查对应终端输出。 ### Node 版本不对 @@ -181,9 +266,11 @@ npm run start:tui -- --resume ```bash node -v +# 一键部署： +./deploy.sh doctor ``` -升级到 Node.js 22 或更高版本后，重新执行 `npm install`。 +升级到 Node.js 22 或更高版本后重试。一键部署也可让 `./deploy.sh` 在确认后协助安装；手动路径则重新执行 `npm install`。 ### 页面打不开 @@ -191,9 +278,9 @@ node -v 处理： -- 确认 `npm run start:web` 还在运行（正式态不要开 `dev`）。 -- 检查 3000 端口是否被占用。 -- 如果 3000 被占用，查看终端输出中的实际前端端口。 +- 先执行 `./deploy.sh status`（手动路径则确认 `npm run start` 仍在运行）。正式态不要开 `dev`。 +- 检查 3000 端口是否被占用；若部署时改过端口，以脚本打印的 Web 地址为准。 +- 若进程已停止：`./deploy.sh start`。 ### 后端未启动 @@ -202,6 +289,7 @@ node -v 处理： ```bash +./deploy.sh status curl http://127.0.0.1:8787/healthz curl http://127.0.0.1:8787/ready ``` @@ -209,7 +297,8 @@ curl http://127.0.0.1:8787/ready 如果健康检查失败： ```bash -npm run start:api +./deploy.sh start +# 手动路径：npm run start ``` ### 注册收不到邮件 diff --git a/package.json b/package.json index feaad081..6dabe4ab 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,9 @@ "dev:web": "npm --workspace @datafoundry/web run dev", "build:web": "npm --workspace @datafoundry/web run build", "start:web": "npm --prefix apps/web run start", + "start": "node scripts/start.mjs", + "test:deploy": "node --test scripts/deploy/*.test.mjs scripts/stack-runtime-config.test.mjs", + "smoke:native-deploy": "node scripts/deploy/smoke-native-deploy.mjs", "test:web": "npm --workspace @datafoundry/web run test", "dev:tui": "npm --workspace @datafoundry/tui run dev", "start:tui": "npm --workspace @datafoundry/tui run start --", diff --git a/scripts/deploy/args.mjs b/scripts/deploy/args.mjs new file mode 100644 index 00000000..93ebf48e --- /dev/null +++ b/scripts/deploy/args.mjs @@ -0,0 +1,87 @@ +const COMMANDS = new Set([ + "deploy", + "start", + "stop", + "restart", + "status", + "logs", + "doctor", + "tui", + "help" +]); + +export function parseDeployArgs(argv = []) { + const tokens = [...argv]; + let command = "deploy"; + let reconfigure = false; + let nonInteractive = false; + let runtimeUrl = null; + let sawCommand = false; + + for (let i = 0; i < tokens.length; i += 1) { + const token = tokens[i]; + if (token === "--reconfigure") { + reconfigure = true; + continue; + } + if (token === "--non-interactive") { + nonInteractive = true; + continue; + } + if (token === "--runtime-url") { + const value = tokens[i + 1]; + if (!value || value.startsWith("-")) { + throw new Error("--runtime-url requires a URL value"); + } + runtimeUrl = value; + i += 1; + continue; + } + if (token.startsWith("-")) { + throw new Error(`Unknown flag: ${token}`); + } + if (!COMMANDS.has(token)) { + throw new Error(`Unknown command: ${token}`); + } + if (sawCommand) { + throw new Error(`Unexpected extra command: ${token}`); + } + command = token; + sawCommand = true; + } + + if (reconfigure && nonInteractive) { + throw new Error("--reconfigure and --non-interactive are mutually exclusive"); + } + if (reconfigure && command !== "deploy") { + throw new Error("--reconfigure is only valid with deploy"); + } + if (nonInteractive && command !== "deploy") { + throw new Error("--non-interactive is only valid with deploy"); + } + if (runtimeUrl != null && command !== "tui") { + throw new Error("--runtime-url is only valid with tui"); + } + + return { command, reconfigure, nonInteractive, runtimeUrl }; +} + +export function deploymentHelp() { + return `./deploy.sh [deploy] [--reconfigure | --non-interactive] +./deploy.sh start|stop|restart|status|logs|doctor|tui|help +./deploy.sh tui [--runtime-url ] + +deploy Configure, install, build (including TUI), start, and verify. Existing deployments use a maintenance window. +start Start an existing build. Does not install or build. +stop Stop only the managed DataFoundry process group. Keeps configuration and data. +restart Stop and start an existing build. Does not install or build. +status Read process state and probe actual service health. +logs Show recent runtime logs and continue following them. Ctrl+C does not stop DataFoundry. +doctor Run read-only dependency, configuration, port, permission, disk, and health checks. +tui Optional: start the TUI client in the foreground after deploy (not a managed background service). +help Show this help. + +One-click deploy builds the TUI so it is ready. Start it in another terminal with ./deploy.sh tui +(or npm run start:tui). It does not stay running with the Web/API stack. +`; +} diff --git a/scripts/deploy/args.test.mjs b/scripts/deploy/args.test.mjs new file mode 100644 index 00000000..bce3c4e7 --- /dev/null +++ b/scripts/deploy/args.test.mjs @@ -0,0 +1,82 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { deploymentHelp, parseDeployArgs } from "./args.mjs"; + +test("defaults to deploy", () => { + assert.deepEqual(parseDeployArgs([]), { + command: "deploy", + reconfigure: false, + nonInteractive: false, + runtimeUrl: null + }); +}); + +test("accepts flags before or after deploy", () => { + assert.deepEqual(parseDeployArgs(["--non-interactive"]), { + command: "deploy", + reconfigure: false, + nonInteractive: true, + runtimeUrl: null + }); + assert.deepEqual(parseDeployArgs(["deploy", "--reconfigure"]), { + command: "deploy", + reconfigure: true, + nonInteractive: false, + runtimeUrl: null + }); + assert.deepEqual(parseDeployArgs(["--reconfigure", "deploy"]), { + command: "deploy", + reconfigure: true, + nonInteractive: false, + runtimeUrl: null + }); +}); + +test("supports every lifecycle command", () => { + for (const command of ["start", "stop", "restart", "status", "logs", "doctor", "tui", "help"]) { + assert.equal(parseDeployArgs([command]).command, command); + } +}); + +test("tui accepts optional --runtime-url", () => { + assert.deepEqual(parseDeployArgs(["tui"]), { + command: "tui", + reconfigure: false, + nonInteractive: false, + runtimeUrl: null + }); + assert.deepEqual(parseDeployArgs(["tui", "--runtime-url", "http://127.0.0.1:9000/api/copilotkit"]), { + command: "tui", + reconfigure: false, + nonInteractive: false, + runtimeUrl: "http://127.0.0.1:9000/api/copilotkit" + }); + assert.deepEqual(parseDeployArgs(["--runtime-url", "http://example/api/copilotkit", "tui"]), { + command: "tui", + reconfigure: false, + nonInteractive: false, + runtimeUrl: "http://example/api/copilotkit" + }); +}); + +test("rejects unknown commands and invalid flag combinations", () => { + assert.throws(() => parseDeployArgs(["start", "--reconfigure"]), /--reconfigure is only valid with deploy/); + assert.throws(() => parseDeployArgs(["status", "--non-interactive"]), /--non-interactive is only valid with deploy/); + assert.throws( + () => parseDeployArgs(["deploy", "--reconfigure", "--non-interactive"]), + /mutually exclusive/ + ); + assert.throws(() => parseDeployArgs(["deploy", "--runtime-url", "http://x"]), /--runtime-url is only valid with tui/); + assert.throws(() => parseDeployArgs(["tui", "--runtime-url"]), /--runtime-url requires a URL value/); + assert.throws(() => parseDeployArgs(["wat"]), /Unknown command/); +}); + +test("help lists lifecycle commands including tui and build readiness note", () => { + const help = deploymentHelp(); + for (const command of ["deploy", "start", "stop", "restart", "status", "logs", "doctor", "tui", "help"]) { + assert.match(help, new RegExp(`^${command}\\b`, "m")); + } + assert.match(help, /builds the TUI|build \(including TUI\)/i); + assert.match(help, /\.\/deploy\.sh tui/); + assert.match(help, /does not stay running/i); +}); diff --git a/scripts/deploy/bootstrap.sh b/scripts/deploy/bootstrap.sh new file mode 100755 index 00000000..d277cb99 --- /dev/null +++ b/scripts/deploy/bootstrap.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Shared bootstrap helpers for native deploy (sourced by ./deploy.sh). + +check_supported_system() { + local os_release="${DATAFOUNDRY_OS_RELEASE_FILE:-/etc/os-release}" + if [[ ! -r "${os_release}" ]]; then + echo "Unsupported operating system: ${os_release} is missing. DataFoundry native deploy supports Ubuntu/Debian only." >&2 + exit 1 + fi + # shellcheck disable=SC1090,SC1091 + source "${os_release}" + case "${ID:-}" in + ubuntu|debian) ;; + *) + echo "Unsupported operating system: ${ID:-unknown}. DataFoundry native deploy supports Ubuntu/Debian only." >&2 + exit 1 + ;; + esac + + local arch + arch="${DATAFOUNDRY_UNAME_M:-$(uname -m)}" + case "${arch}" in + x86_64|amd64|aarch64|arm64) ;; + *) + echo "Unsupported architecture: ${arch}. DataFoundry native deploy supports x86_64/amd64 and aarch64/arm64 only." >&2 + exit 1 + ;; + esac +} + +command_is_readonly() { + local token + for token in "$@"; do + case "${token}" in + status|logs|stop|doctor|help) return 0 ;; + esac + done + return 1 +} + +has_non_interactive_flag() { + local token + for token in "$@"; do + [[ "${token}" == "--non-interactive" ]] && return 0 + done + return 1 +} + +node_major_version() { + local version + version="$(node --version 2>/dev/null || true)" + if [[ "${version}" =~ ^v([0-9]+) ]]; then + echo "${BASH_REMATCH[1]}" + return 0 + fi + return 1 +} + +can_install_noninteractive() { + if [[ "$(id -u)" -eq 0 ]]; then + return 0 + fi + if sudo -n true >/dev/null 2>&1; then + return 0 + fi + return 1 +} + +install_node_22() { + local setup + local nodesource_url="https://deb.nodesource.com/setup_22.x" + echo "Node.js 22 is required." + echo "Installer source: ${nodesource_url}" + echo "Commands:" + echo " curl -fsSL ${nodesource_url} -o /tmp/nodesource_setup.sh" + echo " bash /tmp/nodesource_setup.sh" + echo " apt-get install -y nodejs" + + if has_non_interactive_flag "$@"; then + if ! can_install_noninteractive; then + echo "Non-interactive Node.js installation requires root or passwordless sudo." >&2 + exit 1 + fi + else + local answer + read -r -p "Install Node.js 22 from NodeSource now? [y/N]: " answer + case "${answer}" in + y|Y|yes|YES) ;; + *) + echo "Node.js 22 is required. Install it and re-run ./deploy.sh." >&2 + exit 1 + ;; + esac + fi + + setup="$(mktemp)" + trap 'rm -f "${setup}"' RETURN + curl -fsSL "${nodesource_url}" -o "${setup}" + if [[ "$(id -u)" -eq 0 ]]; then + bash "${setup}" + apt-get install -y nodejs + elif has_non_interactive_flag "$@"; then + sudo -n bash "${setup}" + sudo -n apt-get install -y nodejs + else + sudo bash "${setup}" + sudo apt-get install -y nodejs + fi + rm -f "${setup}" + trap - RETURN + + if ! node --version >/dev/null 2>&1 || ! npm --version >/dev/null 2>&1; then + echo "Node.js 22 installation did not produce working node/npm commands." >&2 + exit 1 + fi + local major + major="$(node_major_version || true)" + if [[ -z "${major}" || "${major}" -lt 22 ]]; then + echo "Node.js 22+ is required after installation; found $(node --version 2>/dev/null || echo none)." >&2 + exit 1 + fi +} + +ensure_node_22() { + local major="" + if command -v node >/dev/null 2>&1; then + major="$(node_major_version || true)" + fi + + if [[ -n "${major}" && "${major}" -ge 22 ]]; then + return 0 + fi + + if command_is_readonly "$@"; then + echo "Node.js 22+ is required for this command. Install Node.js 22 and re-run ./deploy.sh $*." >&2 + exit 1 + fi + + if [[ -n "${major}" && "${major}" -lt 22 ]]; then + echo "Unsupported Node.js version: $(node --version). DataFoundry requires Node.js 22.x." >&2 + fi + + install_node_22 "$@" +} diff --git a/scripts/deploy/bootstrap.test.mjs b/scripts/deploy/bootstrap.test.mjs new file mode 100644 index 00000000..c0e38790 --- /dev/null +++ b/scripts/deploy/bootstrap.test.mjs @@ -0,0 +1,163 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { chmod, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const DEPLOY_SH = path.join(ROOT, "deploy.sh"); +const INSTALL_SH = path.join(ROOT, "scripts/deploy/install-dependency.sh"); + +function runBash(args, options = {}) { + return spawnSync("bash", args, { + cwd: ROOT, + encoding: "utf8", + env: { ...process.env, ...(options.env ?? {}) }, + input: options.input, + timeout: options.timeout ?? 10_000 + }); +} + +async function makeFakePath(binaries) { + const dir = await mkdtemp(path.join(os.tmpdir(), "datafoundry-path-")); + for (const [name, body] of Object.entries(binaries)) { + const filePath = path.join(dir, name); + await writeFile(filePath, body, { mode: 0o755 }); + await chmod(filePath, 0o755); + } + return dir; +} + +test("./deploy.sh help delegates without installing when Node 22+ exists", () => { + const result = runBash([DEPLOY_SH, "help"]); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /deploy\s+Configure/); + assert.match(result.stdout, /\bstart\b/); + assert.match(result.stdout, /\bstop\b/); + assert.match(result.stdout, /\brestart\b/); + assert.match(result.stdout, /\bstatus\b/); + assert.match(result.stdout, /\blogs\b/); + assert.match(result.stdout, /\bdoctor\b/); + assert.match(result.stdout, /\bhelp\b/); + assert.doesNotMatch(result.stdout + result.stderr, /Install Node\.js|nodesource|apt-get install/i); +}); + +test("unsupported OS exits 1 with a precise message", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "datafoundry-os-")); + const osRelease = path.join(dir, "os-release"); + await writeFile(osRelease, 'ID=fedora\nVERSION_ID="40"\n'); + const result = runBash([DEPLOY_SH, "help"], { + env: { DATAFOUNDRY_OS_RELEASE_FILE: osRelease } + }); + assert.equal(result.status, 1); + assert.match(result.stderr, /Unsupported operating system: fedora/); +}); + +test("unsupported architecture exits 1 with a precise message", async () => { + const result = runBash([DEPLOY_SH, "help"], { + env: { DATAFOUNDRY_UNAME_M: "ppc64le" } + }); + assert.equal(result.status, 1); + assert.match(result.stderr, /Unsupported architecture: ppc64le/); +}); + +test("Node 20 is rejected before CLI starts", async () => { + const fakePath = await makeFakePath({ + node: "#!/bin/bash\necho v20.11.0\n", + npm: "#!/bin/bash\necho 10.0.0\n", + curl: "#!/bin/bash\necho curl-should-not-run >&2\nexit 99\n", + sudo: "#!/bin/bash\nexit 1\n", + id: "#!/bin/bash\necho 1000\n" + }); + const script = ` +set -euo pipefail +export PATH="${fakePath}:/bin" +hash -r +command -v node +node --version +source "${DEPLOY_SH}" +ensure_node_22 deploy --non-interactive +echo SHOULD_NOT_REACH +`; + const result = runBash(["-c", script], { + env: { + PATH: `${fakePath}:/bin`, + HOME: os.tmpdir() + }, + input: "" + }); + const output = `${result.stdout ?? ""}${result.stderr ?? ""}`; + assert.notEqual(result.status, 0, output); + assert.match(output, /Unsupported Node\.js version|Node\.js 22|passwordless sudo|requires root/i); + assert.doesNotMatch(output, /SHOULD_NOT_REACH/); + assert.doesNotMatch(output, /curl-should-not-run/); +}); + +test("interactive Node installation prints repository and command, then asks once", async () => { + const fakePath = await makeFakePath({ + curl: "#!/bin/bash\nexit 99\n", + sudo: "#!/bin/bash\nexit 1\n", + id: "#!/bin/bash\necho 1000\n" + }); + const result = runBash( + ["-c", `source "${DEPLOY_SH}"; install_node_22 deploy`], + { + env: { + PATH: `${fakePath}:/bin:/usr/bin`, + HOME: os.tmpdir() + }, + input: "n\n" + } + ); + const output = `${result.stdout ?? ""}${result.stderr ?? ""}`; + assert.notEqual(result.status, 0, output); + assert.match(output, /deb\.nodesource\.com\/setup_22\.x/); + assert.match(output, /Node\.js 22 is required/); + assert.match(output, /apt-get install -y nodejs/); + assert.match(output, /re-run \.\/deploy\.sh/); +}); + +test("--non-interactive never reads stdin and fails without root/passwordless sudo", async () => { + const fakePath = await makeFakePath({ + id: "#!/usr/bin/env bash\necho 1000\n", + sudo: "#!/usr/bin/env bash\nexit 1\n", + curl: "#!/usr/bin/env bash\necho unexpectedly-called >&2; exit 99\n" + }); + const result = runBash( + ["-c", `source "${DEPLOY_SH}"; install_node_22 deploy --non-interactive; echo SHOULD_NOT_REACH`], + { + env: { + PATH: `${fakePath}:/bin:/usr/bin`, + HOME: os.tmpdir() + }, + input: "y\ny\ny\n" + } + ); + const output = `${result.stdout ?? ""}${result.stderr ?? ""}`; + assert.equal(result.status, 1, output); + assert.match(output, /root or passwordless sudo/i); + assert.doesNotMatch(output, /unexpectedly-called|SHOULD_NOT_REACH/); +}); + +test("installer accepts only node", () => { + const bad = runBash([INSTALL_SH, "ruby"]); + assert.equal(bad.status, 2); + assert.match(bad.stderr, /Usage: install-dependency\.sh /); +}); + +test("no code path executes curl | bash", async () => { + const bootstrap = path.join(ROOT, "scripts/deploy/bootstrap.sh"); + const sources = [ + await readFile(DEPLOY_SH, "utf8"), + await readFile(bootstrap, "utf8"), + await readFile(INSTALL_SH, "utf8") + ]; + for (const source of sources) { + assert.doesNotMatch(source, /curl[^\n]*\|\s*(bash|sh)/); + } + // Node bootstrap and dependency installer download to a temp file first. + assert.match(await readFile(bootstrap, "utf8"), /mktemp/); + assert.match(await readFile(INSTALL_SH, "utf8"), /mktemp/); +}); diff --git a/scripts/deploy/cli.mjs b/scripts/deploy/cli.mjs new file mode 100644 index 00000000..041e8f62 --- /dev/null +++ b/scripts/deploy/cli.mjs @@ -0,0 +1,719 @@ +import { spawn } from "node:child_process"; +import { constants, existsSync } from "node:fs"; +import { access, appendFile, copyFile, mkdir, readFile, statfs, writeFile } from "node:fs/promises"; +import { createInterface } from "node:readline"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { deploymentHelp, parseDeployArgs } from "./args.mjs"; +import { + ensureDeploymentEnvironment, + isCompleteDeploymentConfig, + parseDeploymentEnvironment, + redactSensitiveText, + renderWebEnvironment, + updateDeploymentEnvironment, + writeDeploymentConfiguration +} from "./config.mjs"; +import { createDeploymentController } from "./controller.mjs"; +import { ensureDependencies, inspectDependencies } from "./dependencies.mjs"; +import { collectDeploymentHealth, probeHttp, waitForDeployment } from "./health.mjs"; +import { probePort, selectDeploymentPort, verifySelectedPorts } from "./ports.mjs"; +import { + deploymentPaths, + isProcessAlive, + readDeploymentState, + startManagedStack, + stopManagedStack, + verifyManagedProcessForStop, + writeDeploymentState +} from "./process-state.mjs"; + +const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const TUI_ENTRY = "apps/tui/dist/index.js"; + +export function resolveTuiRuntimeUrl(env = {}, runtimeUrl = null) { + const explicit = String(runtimeUrl ?? "").trim(); + if (explicit) return explicit; + const port = Number(env.API_PORT || 8787); + return `http://127.0.0.1:${port}/api/copilotkit`; +} + +export async function inspectWritablePath(targetPath) { + let candidate = path.resolve(targetPath); + while (true) { + try { + await access(candidate, constants.W_OK); + return { writable: true, checkedPath: candidate, exists: existsSync(targetPath) }; + } catch (error) { + const parent = path.dirname(candidate); + if (parent === candidate) { + return { + writable: false, + checkedPath: candidate, + exists: existsSync(targetPath), + error: error?.message ?? String(error) + }; + } + candidate = parent; + } + } +} + +function overlayProcessEnv(text, processEnv = {}) { + const updates = {}; + for (const key of [ + "WEB_HOST", + "WEB_PORT", + "API_HOST", + "API_PORT", + "AUTH_PUBLIC_BASE_URL", + "AUTH_SESSION_SECRET", + "SECRET_MASTER_KEY" + ]) { + if (processEnv[key] != null && String(processEnv[key]).trim() !== "") { + updates[key] = String(processEnv[key]); + } + } + return Object.keys(updates).length > 0 ? updateDeploymentEnvironment(text, updates) : text; +} + +export function collectManagedPorts(env = {}, state = null) { + const ports = new Set(); + for (const key of ["WEB_PORT", "API_PORT"]) { + const port = Number(env[key]); + if (Number.isInteger(port) && port > 0) ports.add(port); + } + if (state?.ports && typeof state.ports === "object") { + for (const value of Object.values(state.ports)) { + const port = Number(value); + if (Number.isInteger(port) && port > 0) ports.add(port); + } + } + return ports; +} + +export function resolveAuthPublicBaseUrl(existing, oldWebPort, newWebPort) { + const url = String(existing ?? "").trim(); + if (!url) return `http://127.0.0.1:${newWebPort}`; + + try { + const parsed = new URL(url); + const isLoopback = /^(127\.0\.0\.1|localhost)$/i.test(parsed.hostname); + const explicitPort = parsed.port; + + if (isLoopback && url.includes(`:${oldWebPort}`)) { + return url.replace(`:${oldWebPort}`, `:${newWebPort}`); + } + if (!explicitPort && !isLoopback) { + return url; + } + if (isLoopback) { + return `http://127.0.0.1:${newWebPort}`; + } + return url; + } catch { + return `http://127.0.0.1:${newWebPort}`; + } +} + +function formatDeployLogTimestamp(date = new Date()) { + return date.toISOString().replace(/[-:TZ.]/g, "").slice(0, 14); +} + +export async function createDeployLogWriter(root, options = {}) { + const logsDir = path.join(root, "storage/logs"); + await mkdir(logsDir, { recursive: true }); + const stamp = options.timestamp ?? formatDeployLogTimestamp(); + const logFile = path.join(logsDir, `deploy-${stamp}.log`); + const latestFile = path.join(logsDir, "deploy-latest.log"); + const relativeLogPath = path.join("storage/logs", `deploy-${stamp}.log`); + + await writeFile(logFile, `[${new Date().toISOString()}] deploy started\n`, { + encoding: "utf8", + mode: 0o600 + }); + + let pendingLine = ""; + + async function flushBufferedLines(text, { finalize = false } = {}) { + pendingLine += String(text ?? ""); + let ready = ""; + + while (true) { + const newlineIndex = pendingLine.indexOf("\n"); + if (newlineIndex === -1) break; + ready += pendingLine.slice(0, newlineIndex + 1); + pendingLine = pendingLine.slice(newlineIndex + 1); + } + + if (finalize && pendingLine.length > 0) { + ready += pendingLine; + pendingLine = ""; + } + + if (ready.length > 0) { + await appendFile(logFile, redactSensitiveText(ready), { encoding: "utf8", mode: 0o600 }); + } + } + + return { + logPath: relativeLogPath, + async append(text) { + await flushBufferedLines(text); + }, + async finalize() { + await flushBufferedLines("", { finalize: true }); + await copyFile(logFile, latestFile); + } + }; +} + +const MAX_PORT_MENU_INVALID_CHOICES = 5; + +async function selectPortWithHint(options) { + return selectDeploymentPort({ + ...options, + ask: async (prompt) => { + if (!prompt.includes("请选择")) { + return options.ask(prompt); + } + + for (let attempt = 0; attempt < MAX_PORT_MENU_INVALID_CHOICES; attempt += 1) { + const answer = await options.ask(prompt); + if (!answer || ["1", "2", ""].includes(String(answer).trim())) { + return answer; + } + options.print("请输入 1 或 2"); + } + + throw new Error("port selection cancelled: too many invalid choices"); + } + }); +} + +export async function configureDeploymentInteractively(options) { + const { + root, + sourceText, + reconfigure, + nonInteractive, + ask, + print, + probe, + processEnv = {}, + write = false, + timestamp + } = options; + + let text = overlayProcessEnv(sourceText ?? "", processEnv); + const originalEnv = parseDeploymentEnvironment(text); + const completeBeforeFill = isCompleteDeploymentConfig(originalEnv); + let ensured = ensureDeploymentEnvironment(text); + text = ensured.text; + let env = ensured.env; + const oldWebPort = env.WEB_PORT; + + if (!reconfigure && !nonInteractive && completeBeforeFill && sourceText?.trim()) { + return { env, envText: text, webText: renderWebEnvironment(env), wrote: false }; + } + + const deploymentState = root ? await readDeploymentState(root).catch(() => null) : null; + const managedPorts = collectManagedPorts(env, deploymentState); + + const reserved = new Set(); + const webPort = await selectPortWithHint({ + label: "Web", + defaultPort: Number(env.WEB_PORT || 3000), + reserved, + managedPorts, + nonInteractive, + ask, + print, + probe + }); + reserved.add(webPort); + const apiPort = await selectPortWithHint({ + label: "API", + defaultPort: Number(env.API_PORT || 8787), + reserved, + managedPorts, + nonInteractive, + ask, + print, + probe + }); + reserved.add(apiPort); + + const updates = { + WEB_PORT: String(webPort), + API_PORT: String(apiPort), + AUTH_PUBLIC_BASE_URL: resolveAuthPublicBaseUrl(env.AUTH_PUBLIC_BASE_URL, oldWebPort, webPort) + }; + + if (!nonInteractive) { + const defaultUrl = updates.AUTH_PUBLIC_BASE_URL; + const answer = String((await ask(`浏览器公开访问地址 [${defaultUrl}]：`)) ?? "").trim(); + if (answer) { + let url; + try { + url = new URL(answer); + } catch { + throw new Error("AUTH_PUBLIC_BASE_URL must be a valid URL"); + } + const urlPort = url.port || (url.protocol === "https:" ? "443" : "80"); + if (String(urlPort) !== String(webPort)) { + const confirm = String( + (await ask(`公开地址端口 ${urlPort} 与 Web 端口 ${webPort} 不一致。确认为反向代理？[y/N]：`)) ?? "" + ) + .trim() + .toLowerCase(); + if (confirm !== "y" && confirm !== "yes") { + throw new Error("Public URL port must match the selected Web port"); + } + } + updates.AUTH_PUBLIC_BASE_URL = answer; + } + } + + text = updateDeploymentEnvironment(text, updates); + ensured = ensureDeploymentEnvironment(text); + text = ensured.text; + env = ensured.env; + + if ( + (env.WEB_HOST === "0.0.0.0" || env.WEB_HOST === "::") && + /^https?:\/\/(127\.0\.0\.1|localhost)(:|\/|$)/i.test(env.AUTH_PUBLIC_BASE_URL) + ) { + print("警告：WEB_HOST 对外绑定，但 AUTH_PUBLIC_BASE_URL 仅适合本机访问。远程访问请设置正确的公开地址。"); + } + + const webText = renderWebEnvironment(env); + let wrote = false; + if (write) { + await writeDeploymentConfiguration(root, text, webText, { + backup: Boolean(reconfigure), + timestamp + }); + wrote = true; + } + + return { env, envText: text, webText, wrote }; +} + +function createReadlineAsk() { + const rl = createInterface({ input: process.stdin, output: process.stdout }); + const ask = (prompt) => + new Promise((resolve) => { + rl.question(prompt, (answer) => resolve(answer)); + }); + ask.close = () => rl.close(); + return ask; +} + +async function loadRootEnvText(root) { + try { + return await readFile(path.join(root, ".env"), "utf8"); + } catch (error) { + if (error?.code === "ENOENT") return ""; + throw error; + } +} + +function runCommand(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: options.cwd ?? ROOT, + env: options.env ?? process.env, + stdio: options.log ? ["ignore", "pipe", "pipe"] : (options.stdio ?? "inherit") + }); + + if (options.log) { + const tee = (stream, target) => { + stream.on("data", (chunk) => { + target.write(chunk); + options.log.append(chunk).catch(() => {}); + }); + }; + tee(child.stdout, process.stdout); + tee(child.stderr, process.stderr); + } + + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) resolve(); + else { + const error = new Error(`${command} ${args.join(" ")} failed with code ${code}`); + if (options.logPath) error.logPath = options.logPath; + reject(error); + } + }); + }); +} + +function createRealDeps(context) { + const { root, parsed, ask, print } = context; + let config = null; + let deployLog = null; + + const commandOptions = () => ({ + cwd: root, + log: deployLog ?? undefined, + logPath: deployLog?.logPath + }); + + return { + async beginDeployLog() { + deployLog = await createDeployLogWriter(root); + return deployLog.logPath; + }, + async finalizeDeployLog() { + if (deployLog) await deployLog.finalize(); + }, + async loadConfiguration() { + const sourceText = await loadRootEnvText(root); + const overlaid = overlayProcessEnv(sourceText, process.env); + const generateSecrets = parsed.command === "deploy"; + const ensured = ensureDeploymentEnvironment(overlaid, { generateSecrets }); + config = { env: ensured.env, envText: ensured.text, sourceText, generatedKeys: ensured.generatedKeys }; + return config; + }, + async preflight(current, options) { + if (options?.command === "start") { + const apiDist = path.join(root, "apps/api/dist/index.js"); + const webBuild = path.join(root, "apps/web/.next/BUILD_ID"); + const webEnvLocal = path.join(root, "apps/web/.env.local"); + if (!existsSync(apiDist) || !existsSync(webBuild) || !existsSync(webEnvLocal)) { + throw new Error("请先运行 ./deploy.sh deploy"); + } + const diskEnv = parseDeploymentEnvironment(current.sourceText ?? ""); + if (!isCompleteDeploymentConfig(diskEnv)) { + throw new Error("缺少合法完整的 .env 配置，请先运行 ./deploy.sh deploy"); + } + } + }, + async configure(current) { + const configured = await configureDeploymentInteractively({ + root, + sourceText: current.sourceText ?? "", + reconfigure: parsed.reconfigure, + nonInteractive: parsed.nonInteractive, + ask, + print, + processEnv: process.env, + write: false + }); + config = { ...current, ...configured }; + return config; + }, + async checkDependencies() { + await ensureDependencies({ + nonInteractive: parsed.nonInteractive, + ask, + print + }); + }, + async selectPorts(current) { + return current; + }, + async writeConfiguration(current) { + await writeDeploymentConfiguration(root, current.envText, current.webText, { + backup: parsed.reconfigure + }); + }, + async isRunning() { + const state = await readDeploymentState(root); + return Boolean(state?.pid && isProcessAlive(state.pid)); + }, + async stop() { + await stopManagedStack(root); + }, + async installProject() { + await runCommand("npm", ["ci"], commandOptions()); + }, + async buildTypeScript() { + await runCommand("npm", ["run", "build"], commandOptions()); + }, + async buildWeb() { + await runCommand("npm", ["run", "build:web"], commandOptions()); + }, + async buildTui() { + await runCommand("npm", ["run", "build:tui"], commandOptions()); + const entry = path.join(root, TUI_ENTRY); + if (!existsSync(entry)) { + throw new Error(`TUI build did not produce ${TUI_ENTRY}`); + } + }, + async ensureTuiReady() { + const entry = path.join(root, TUI_ENTRY); + if (!existsSync(entry)) { + throw new Error(`TUI is not built (${TUI_ENTRY}). Run ./deploy.sh deploy first.`); + } + }, + async checkApiForTui(current) { + const apiBaseUrl = `http://127.0.0.1:${current.env.API_PORT}`; + const health = await probeHttp(`${apiBaseUrl}/healthz`); + if (!(health.ok && health.status === 200)) { + throw new Error( + `API is not healthy at ${apiBaseUrl}/healthz. Start the stack with ./deploy.sh start (or deploy), then retry ./deploy.sh tui.` + ); + } + }, + async startTui(current, parsed) { + const runtimeUrl = resolveTuiRuntimeUrl(current.env, parsed?.runtimeUrl); + print(`Starting TUI against ${runtimeUrl} (foreground client; Ctrl+C exits TUI only).`); + await runCommand("npm", ["run", "start:tui", "--", "--runtime-url", runtimeUrl], { + cwd: root, + stdio: "inherit" + }); + }, + + async verifyPorts(current) { + const services = [ + { label: "Web", port: current.env.WEB_PORT }, + { label: "API", port: current.env.API_PORT } + ]; + // After stop-old, managed ports may still be briefly held by our previous process. + // Treat them as owned so verify-ports-again does not false-fail during redeploy. + const state = await readDeploymentState(root).catch(() => null); + const managedPorts = collectManagedPorts(current.env, state); + await verifySelectedPorts(services, { managedPorts }); + }, + async start(current) { + await startManagedStack(root, { + env: { ...process.env, ...current.env }, + ports: { + web: Number(current.env.WEB_PORT), + api: Number(current.env.API_PORT) + } + }); + }, + async waitForHealth(current) { + try { + return await waitForDeployment({ + checkProcessAlive: async () => { + const state = await readDeploymentState(root); + return Boolean(state?.pid && isProcessAlive(state.pid)); + }, + apiBaseUrl: `http://127.0.0.1:${current.env.API_PORT}`, + webUrl: `http://127.0.0.1:${current.env.WEB_PORT}` + }); + } catch (error) { + const latest = await readDeploymentState(root); + if (latest?.pid) { + await writeDeploymentState(root, { ...latest, status: "unhealthy" }); + } + throw error; + } + }, + async markHealthy(current) { + const state = await readDeploymentState(root); + if (!state?.pid || !isProcessAlive(state.pid)) { + throw new Error("Managed process is not alive; refusing to mark deployment healthy"); + } + if (process.platform === "linux") { + const verification = await verifyManagedProcessForStop(state.pid, state.launchId); + if (!verification.allowed) { + throw new Error( + `Refusing to mark healthy: launch marker does not identify DataFoundry (${verification.reason})` + ); + } + } + await writeDeploymentState(root, { ...state, status: "healthy" }); + if (deployLog) await deployLog.finalize(); + print(`DataFoundry is healthy at ${current.env.AUTH_PUBLIC_BASE_URL}`); + print("Next: open Web, register/login, then create and enable a model profile."); + print("TUI is built and ready. Start it in another terminal (foreground client, not a background service):"); + print(" ./deploy.sh tui"); + print(" # or: npm run start:tui"); + }, + + async status() { + const state = await readDeploymentState(root); + const env = + config?.env ?? + ensureDeploymentEnvironment(await loadRootEnvText(root), { generateSecrets: false }).env; + const summary = await collectDeploymentHealth({ + checkProcessAlive: async () => Boolean(state?.pid && isProcessAlive(state.pid)), + apiBaseUrl: `http://127.0.0.1:${env.API_PORT}`, + webUrl: `http://127.0.0.1:${env.WEB_PORT}` + }); + print(`进程 ${summary.process}`); + print(`API ${summary.apiHealth}/${summary.apiReady}`); + print(`Web ${summary.web}`); + return summary; + }, + async logs() { + const logPath = deploymentPaths(root).runtimeLog; + await mkdir(path.dirname(logPath), { recursive: true }); + if (!existsSync(logPath)) await runCommand("touch", [logPath]); + await runCommand("tail", ["-n", "200", "-F", logPath]); + }, + async doctor() { + await runDeploymentDoctor(root, { print }); + }, + async printHelp(text) { + print(text); + }, + async reportFailure(failure) { + if (deployLog) await deployLog.finalize().catch(() => {}); + for (const line of formatDeploymentFailure(failure).split("\n")) { + print(line); + } + } + }; +} + +export async function runDeploymentDoctor(root, options = {}) { + const print = options.print ?? (() => {}); + const lines = []; + const record = (line) => { + lines.push(line); + print(line); + }; + + record(`os: ${process.platform} ${os.release()}`); + record(`arch: ${os.arch()}`); + + const envText = await loadRootEnvText(root).catch(() => ""); + const ensured = ensureDeploymentEnvironment(envText, { generateSecrets: false }); + const env = ensured.env; + + const deps = await inspectDependencies({ run: options.run }); + for (const entry of deps) { + record( + `dependency ${entry.name}: ${entry.status}${entry.foundVersion ? ` (${entry.foundVersion})` : ""}` + ); + } + + const configIssues = []; + if (!env.WEB_PORT) configIssues.push("WEB_PORT missing"); + if (!env.API_PORT) configIssues.push("API_PORT missing"); + if (!env.AUTH_PUBLIC_BASE_URL) configIssues.push("AUTH_PUBLIC_BASE_URL missing"); + record(`config: ${configIssues.length === 0 ? "ok" : configIssues.join(", ")}`); + record(`config web=${env.WEB_PORT} api=${env.API_PORT} public=${env.AUTH_PUBLIC_BASE_URL}`); + + const state = await readDeploymentState(root); + const managedPorts = collectManagedPorts(env, state); + const portServices = [ + { label: "web", port: env.WEB_PORT }, + { label: "api", port: env.API_PORT } + ]; + + const probe = options.probe ?? ((host, port) => probePort(host, port)); + for (const service of portServices) { + const port = Number(service.port); + if (!Number.isInteger(port)) { + record(`port ${service.label}: invalid (${service.port})`); + continue; + } + const result = await probe("0.0.0.0", port); + if (result.available) { + record(`port ${service.label} ${port}: available`); + } else if (managedPorts.has(port)) { + record(`port ${service.label} ${port}: in-use (managed)`); + } else { + record(`port ${service.label} ${port}: in-use (${result.owner ?? "unknown"})`); + } + } + + const storageRoot = path.join(root, env.STORAGE_ROOT_DIR || "storage"); + const envPath = path.join(root, ".env"); + const storageWritability = await inspectWritablePath(storageRoot); + if (storageWritability.writable) { + record( + `permissions storage: writable (${storageWritability.exists ? storageRoot : storageWritability.checkedPath})` + ); + } else { + record( + `permissions storage: not writable (${storageWritability.error ?? storageWritability.checkedPath})` + ); + } + record(`permissions .env: ${existsSync(envPath) ? "present" : "missing"}`); + + try { + const diskPath = existsSync(storageRoot) ? storageRoot : root; + const stats = await statfs(diskPath); + const freeGiB = ((stats.bfree * stats.bsize) / (1024 ** 3)).toFixed(1); + record(`disk free: ${freeGiB} GiB (${diskPath})`); + } catch (error) { + record(`disk: unavailable (${error.message})`); + } + + if (state?.pid) { + const alive = isProcessAlive(state.pid); + record(`pid: ${state.pid} status=${state.status ?? "unknown"} alive=${alive}`); + } else { + record("pid: none"); + } + + if (state?.pid && isProcessAlive(state.pid)) { + const summary = await collectDeploymentHealth( + { + processAlive: true, + apiBaseUrl: `http://127.0.0.1:${env.API_PORT}`, + webUrl: `http://127.0.0.1:${env.WEB_PORT}` + }, + options.healthOptions + ); + record( + `health process=${summary.process} api=${summary.apiHealth}/${summary.apiReady} web=${summary.web}` + ); + } else { + record("health: skipped (process not running)"); + } + + return { lines, env, state }; +} + +export function formatDeploymentFailure(failure) { + const summary = failure.summary ?? failure.error?.message ?? failure.stage; + const lines = [`✗ ${summary}`]; + if (failure.maintenanceWindow) { + lines.push("更新处于维护窗口，服务当前已停止；现有数据未被修改。"); + } else if (failure.configurationChanged) { + lines.push("配置可能已更新；现有数据未被修改。"); + } else { + lines.push("现有数据未被修改。"); + } + lines.push(`完整日志：${failure.logPath}`); + lines.push(`修复后重试：${failure.retryCommand}`); + lines.push(`诊断命令：${failure.doctorCommand}`); + if (failure.error?.message && failure.error.message !== summary) { + lines.push(redactSensitiveText(failure.error.message)); + } + return lines.join("\n"); +} + +export async function main(argv = process.argv.slice(2), options = {}) { + const root = options.root ?? ROOT; + let parsed; + try { + parsed = parseDeployArgs(argv); + } catch (error) { + process.stderr.write(`${error.message}\n`); + return 2; + } + + const print = options.print ?? ((message) => process.stdout.write(`${message}\n`)); + const ask = options.ask ?? createReadlineAsk(); + try { + if (parsed.command === "help") { + print(deploymentHelp()); + return 0; + } + const controller = createDeploymentController(createRealDeps({ root, parsed, ask, print })); + await controller.run(parsed); + return 0; + } catch (error) { + if (!error?.failure) print(redactSensitiveText(error.message ?? String(error))); + return 1; + } finally { + ask.close?.(); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + process.exitCode = await main(process.argv.slice(2)); +} diff --git a/scripts/deploy/cli.test.mjs b/scripts/deploy/cli.test.mjs new file mode 100644 index 00000000..b48921ed --- /dev/null +++ b/scripts/deploy/cli.test.mjs @@ -0,0 +1,372 @@ +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { mkdtemp, readdir, readFile, writeFile, mkdir } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { + configureDeploymentInteractively, + collectManagedPorts, + createDeployLogWriter, + inspectWritablePath, + resolveAuthPublicBaseUrl, + resolveTuiRuntimeUrl, + runDeploymentDoctor +} from "./cli.mjs"; + +function createAsk(answers) { + const queue = [...answers]; + return async () => { + if (queue.length === 0) throw new Error("unexpected prompt"); + return queue.shift(); + }; +} + +test("first run selects Web and API ports with defaults", async () => { + const lines = []; + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const result = await configureDeploymentInteractively({ + root, + sourceText: "", + reconfigure: false, + nonInteractive: false, + ask: createAsk(["1", "1", ""]), + print: (line) => lines.push(String(line)), + probe: async () => ({ available: true, owner: null }) + }); + assert.equal(result.env.WEB_PORT, "3000"); + assert.equal(result.env.API_PORT, "8787"); +}); + +test("existing complete config skips prompts unless reconfigure", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const source = [ + "WEB_PORT=3310", + "API_PORT=8877", + "AUTH_SESSION_SECRET=existing-session-secret-value", + "SECRET_MASTER_KEY=existing-master-secret-value", + "AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3310", + "DATAFOUNDRY_AUTH_MODE=password", + "AUTH_EMAIL_DELIVERY=test", + "WEB_HOST=0.0.0.0", + "API_HOST=127.0.0.1", + "STORAGE_ROOT_DIR=storage", + "METADATA_DB_PATH=storage/metadata/workbench.sqlite" + ].join("\n"); + let asked = 0; + const result = await configureDeploymentInteractively({ + root, + sourceText: source, + reconfigure: false, + nonInteractive: false, + ask: async () => { + asked += 1; + throw new Error("must not prompt"); + }, + print: () => {}, + probe: async () => ({ available: true, owner: null }) + }); + assert.equal(asked, 0); + assert.equal(result.env.WEB_PORT, "3310"); +}); + +test("partial .env is not treated as complete before fill and still prompts", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-partial-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + let asked = 0; + const result = await configureDeploymentInteractively({ + root, + sourceText: "FOO=bar\nCUSTOM=keep-me\n", + reconfigure: false, + nonInteractive: false, + ask: createAsk(["1", "1", ""]), + print: () => { + asked += 1; + }, + probe: async () => ({ available: true, owner: null }) + }); + assert.match(result.envText, /CUSTOM=keep-me/); + assert.ok(asked > 0); +}); + +test("non-interactive never calls ask and warns on loopback public URL with remote bind", async () => { + const lines = []; + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const result = await configureDeploymentInteractively({ + root, + sourceText: "", + reconfigure: false, + nonInteractive: true, + processEnv: { WEB_HOST: "0.0.0.0" }, + ask: async () => assert.fail("must not prompt"), + print: (line) => lines.push(String(line)), + probe: async () => ({ available: true, owner: null }) + }); + assert.equal(result.env.WEB_PORT, "3000"); + assert.match(lines.join("\n"), /本机访问|local-machine|仅适合本机/i); +}); + +test("port menu rejects n with explicit hint", async () => { + const lines = []; + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const ask = createAsk(["n", "1", "1", ""]); + await configureDeploymentInteractively({ + root, + sourceText: "", + reconfigure: false, + nonInteractive: false, + ask, + print: (line) => lines.push(String(line)), + probe: async () => ({ available: true, owner: null }) + }); + assert.match(lines.join("\n"), /请输入 1 或 2/); +}); + +test("port menu aborts after too many invalid choices", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + await assert.rejects( + () => + configureDeploymentInteractively({ + root, + sourceText: "", + reconfigure: false, + nonInteractive: false, + ask: createAsk(["n", "n", "n", "n", "n"]), + print: () => {}, + probe: async () => ({ available: true, owner: null }) + }), + /too many invalid choices/ + ); +}); + +test("reconfigure creates backup and keeps secrets", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const source = [ + "WEB_PORT=3000", + "API_PORT=8787", + "AUTH_SESSION_SECRET=existing-session-secret-value", + "SECRET_MASTER_KEY=existing-master-secret-value", + "AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3000", + "DATAFOUNDRY_AUTH_MODE=password", + "AUTH_EMAIL_DELIVERY=test", + "WEB_HOST=0.0.0.0", + "API_HOST=127.0.0.1", + "STORAGE_ROOT_DIR=storage", + "METADATA_DB_PATH=storage/metadata/workbench.sqlite" + ].join("\n"); + await writeFile(path.join(root, ".env"), `${source}\n`); + const result = await configureDeploymentInteractively({ + root, + sourceText: source, + reconfigure: true, + nonInteractive: false, + ask: createAsk(["1", "1", ""]), + print: () => {}, + probe: async () => ({ available: true, owner: null }), + write: true, + timestamp: "20260722-150000" + }); + assert.match(result.envText, /AUTH_SESSION_SECRET=existing-session-secret-value/); + assert.equal( + await readFile(path.join(root, ".env.backup-20260722-150000"), "utf8"), + `${source}\n` + ); +}); + +test("collectManagedPorts gathers env and deployment state ports", () => { + const ports = collectManagedPorts( + { WEB_PORT: "3310", API_PORT: "8877" }, + { ports: { web: 3310, api: 8877 } } + ); + assert.deepEqual([...ports].sort((a, b) => a - b), [3310, 8877]); +}); + +test("reconfigure reuses managed listening ports during update preflight", async () => { + const probes = []; + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + await mkdir(path.join(root, "storage/run"), { recursive: true }); + const source = [ + "WEB_PORT=3310", + "API_PORT=8877", + "AUTH_SESSION_SECRET=existing-session-secret-value", + "SECRET_MASTER_KEY=existing-master-secret-value", + "AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3310", + "DATAFOUNDRY_AUTH_MODE=password", + "AUTH_EMAIL_DELIVERY=test", + "WEB_HOST=0.0.0.0", + "API_HOST=127.0.0.1", + "STORAGE_ROOT_DIR=storage", + "METADATA_DB_PATH=storage/metadata/workbench.sqlite" + ].join("\n"); + await writeFile( + path.join(root, "storage/run/deployment.json"), + `${JSON.stringify({ + pid: 99999, + launchId: "launch-1", + status: "healthy", + startedAt: "2026-07-22T00:00:00.000Z", + ports: { web: 3310, api: 8877 } + })}\n` + ); + const result = await configureDeploymentInteractively({ + root, + sourceText: source, + reconfigure: true, + nonInteractive: true, + ask: async () => assert.fail("must not prompt"), + print: () => {}, + probe: async (port) => { + probes.push(port); + return { available: false, owner: "datafoundry pid=99999" }; + } + }); + assert.equal(result.env.WEB_PORT, "3310"); + assert.equal(result.env.API_PORT, "8877"); + assert.ok(probes.includes(3310)); + assert.ok(probes.includes(8877)); +}); + +test("resolveAuthPublicBaseUrl preserves HTTPS proxy URLs without explicit port", () => { + assert.equal( + resolveAuthPublicBaseUrl("https://prod.example.com", "3000", "3001"), + "https://prod.example.com" + ); + assert.equal( + resolveAuthPublicBaseUrl("http://127.0.0.1:3000", "3000", "3001"), + "http://127.0.0.1:3001" + ); + assert.equal( + resolveAuthPublicBaseUrl("https://prod.example.com:8443", "8443", "3001"), + "https://prod.example.com:8443" + ); + assert.equal( + resolveAuthPublicBaseUrl("https://prod.example.com:3000", "3000", "3001"), + "https://prod.example.com:3000" + ); + assert.equal( + resolveAuthPublicBaseUrl("http://example.com:3000", "3000", "3310"), + "http://example.com:3000" + ); +}); + +test("reconfigure keeps HTTPS public URL when ports are unchanged", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const source = [ + "WEB_PORT=3000", + "API_PORT=8787", + "AUTH_SESSION_SECRET=existing-session-secret-value", + "SECRET_MASTER_KEY=existing-master-secret-value", + "AUTH_PUBLIC_BASE_URL=https://prod.example.com", + "DATAFOUNDRY_AUTH_MODE=password", + "AUTH_EMAIL_DELIVERY=test", + "WEB_HOST=0.0.0.0", + "API_HOST=127.0.0.1", + "STORAGE_ROOT_DIR=storage", + "METADATA_DB_PATH=storage/metadata/workbench.sqlite" + ].join("\n"); + const result = await configureDeploymentInteractively({ + root, + sourceText: source, + reconfigure: true, + nonInteractive: true, + ask: async () => assert.fail("must not prompt"), + print: () => {}, + probe: async () => ({ available: true, owner: null }) + }); + assert.equal(result.env.AUTH_PUBLIC_BASE_URL, "https://prod.example.com"); +}); + +test("createDeployLogWriter redacts secrets and updates deploy-latest.log", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-log-")); + const writer = await createDeployLogWriter(root, { timestamp: "20260722143000" }); + await writer.append("AUTH_SESSION_SECRET=fixture-deploy-secret-at-least-32-chars\nnpm ci output\n"); + await writer.finalize(); + const logText = await readFile(path.join(root, "storage/logs/deploy-20260722143000.log"), "utf8"); + const latestText = await readFile(path.join(root, "storage/logs/deploy-latest.log"), "utf8"); + assert.doesNotMatch(logText, /fixture-deploy-secret-at-least-32-chars/); + assert.match(logText, /npm ci output/); + assert.equal(latestText, logText); + assert.equal(writer.logPath, "storage/logs/deploy-20260722143000.log"); +}); + +test("createDeployLogWriter redacts secrets split across append chunks", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-log-chunk-")); + const writer = await createDeployLogWriter(root, { timestamp: "20260722143001" }); + await writer.append("AUTH_SESSION_SECRET=fixture-deploy-secret-"); + await writer.append("at-least-32-chars\nnpm ci output\n"); + await writer.finalize(); + const logText = await readFile(path.join(root, "storage/logs/deploy-20260722143001.log"), "utf8"); + assert.doesNotMatch(logText, /fixture-deploy-secret-at-least-32-chars/); + assert.match(logText, /AUTH_SESSION_SECRET=\*{4,8}/); + assert.match(logText, /npm ci output/); +}); + +test("runDeploymentDoctor reports os, deps, config, ports, permissions, disk, pid, and health", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-doctor-")); + await mkdir(path.join(root, "storage"), { recursive: true }); + await writeFile( + path.join(root, ".env"), + [ + "WEB_PORT=3000", + "API_PORT=8787", + "AUTH_SESSION_SECRET=existing-session-secret-value", + "SECRET_MASTER_KEY=existing-master-secret-value", + "AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3000" + ].join("\n") + ); + const beforeEntries = await readdir(path.join(root, "storage")); + const lines = []; + const result = await runDeploymentDoctor(root, { + print: (line) => lines.push(line), + run: async (command, args = []) => { + const key = [command, ...args].join(" "); + if (key === "node --version") return { stdout: "v22.14.0\n" }; + if (key === "npm --version") return { stdout: "10.9.0\n" }; + throw new Error(`missing mock for ${key}`); + }, + probe: async () => ({ available: true, owner: null }) + }); + const joined = lines.join("\n"); + assert.match(joined, /^os: /m); + assert.match(joined, /^arch: /m); + assert.match(joined, /dependency node: ok/); + assert.match(joined, /config: ok/); + assert.match(joined, /port web 3000: available/); + assert.match(joined, /permissions storage: writable/); + assert.match(joined, /disk free:/); + assert.match(joined, /pid: none/); + assert.match(joined, /health: skipped/); + assert.ok(result.lines.length >= 8); + const afterEntries = await readdir(path.join(root, "storage")); + assert.deepEqual(afterEntries, beforeEntries); + assert.ok(afterEntries.every((entry) => !entry.startsWith(".doctor-write-"))); +}); + +test("inspectWritablePath checks access without creating files", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-writable-")); + const missingChild = path.join(root, "storage", "nested"); + const result = await inspectWritablePath(missingChild); + assert.equal(result.writable, true); + assert.equal(result.exists, false); + assert.equal(existsSync(missingChild), false); + assert.equal(existsSync(path.join(root, "storage")), false); +}); + +test("resolveTuiRuntimeUrl defaults to deployed API port", () => { + assert.equal( + resolveTuiRuntimeUrl({ API_PORT: "8877" }), + "http://127.0.0.1:8877/api/copilotkit" + ); + assert.equal( + resolveTuiRuntimeUrl({ API_PORT: "8877" }, "http://example/api/copilotkit"), + "http://example/api/copilotkit" + ); +}); diff --git a/scripts/deploy/config.mjs b/scripts/deploy/config.mjs new file mode 100644 index 00000000..983dc95c --- /dev/null +++ b/scripts/deploy/config.mjs @@ -0,0 +1,218 @@ +import { randomBytes } from "node:crypto"; +import { mkdir, rename, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; + +/** Minimal dotenv parser — avoids requiring a root `dotenv` dependency for deploy scripts. */ +export function parseDeploymentEnvironment(sourceText = "") { + const env = {}; + for (const rawLine of String(sourceText).split(/\r?\n/u)) { + const line = rawLine.trim(); + if (!line || line.startsWith("#")) continue; + const matched = /^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/u.exec(line); + if (!matched) continue; + let value = matched[2] ?? ""; + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + env[matched[1]] = value; + } + return env; +} + +const DEFAULTS = { + WEB_HOST: "0.0.0.0", + WEB_PORT: "3000", + API_HOST: "127.0.0.1", + API_PORT: "8787", + DATAFOUNDRY_AUTH_MODE: "password", + AUTH_EMAIL_DELIVERY: "test", + AUTH_PUBLIC_BASE_URL: "http://127.0.0.1:3000", + STORAGE_ROOT_DIR: "storage", + METADATA_DB_PATH: "storage/metadata/workbench.sqlite" +}; + +const SECRET_KEYS = ["AUTH_SESSION_SECRET", "SECRET_MASTER_KEY"]; +const PLACEHOLDER_SECRETS = new Set(["", "change-me", "replace-me"]); +const SENSITIVE_KEY_PATTERN = /KEY|SECRET|TOKEN|PASSWORD|COOKIE|AUTHORIZATION/i; +const SENSITIVE_JSON_KEY_PATTERN = /^(?:api[_-]?key|access[_-]?token|refresh[_-]?token|id[_-]?token|client[_-]?secret|secret|token|password|authorization|auth)$/i; + +export function isPlaceholderSecret(value) { + return value == null || PLACEHOLDER_SECRETS.has(String(value).trim()); +} + +export function isCompleteDeploymentConfig(env = {}) { + return Boolean( + String(env.WEB_PORT ?? "").trim() && + String(env.API_PORT ?? "").trim() && + String(env.AUTH_PUBLIC_BASE_URL ?? "").trim() && + !isPlaceholderSecret(env.AUTH_SESSION_SECRET) && + !isPlaceholderSecret(env.SECRET_MASTER_KEY) + ); +} + +function maskSecret(value = "") { + return "*".repeat(Math.min(8, Math.max(4, String(value).length || 4))); +} + +function assertNoNewlines(value, key) { + if (String(value).includes("\n") || String(value).includes("\r")) { + throw new Error(`${key} must not contain newline characters`); + } +} + +function upsertEnvText(sourceText, updates) { + const lines = sourceText.length > 0 ? sourceText.replace(/\r\n/g, "\n").split("\n") : []; + if (lines.length > 0 && lines.at(-1) === "") lines.pop(); + + const seen = new Set(); + const next = lines.map((line) => { + const match = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line); + if (!match) return line; + const key = match[1]; + if (!(key in updates)) return line; + seen.add(key); + assertNoNewlines(updates[key], key); + return `${key}=${updates[key]}`; + }); + + for (const [key, value] of Object.entries(updates)) { + if (seen.has(key)) continue; + assertNoNewlines(value, key); + next.push(`${key}=${value}`); + } + + return `${next.join("\n")}\n`; +} + +export function updateDeploymentEnvironment(sourceText, updates) { + return upsertEnvText(sourceText ?? "", updates); +} + +export function ensureDeploymentEnvironment(sourceText, options = {}) { + const randomSecret = options.randomSecret ?? (() => randomBytes(32).toString("base64url")); + const generateSecrets = options.generateSecrets !== false; + const parsed = parseDeploymentEnvironment(sourceText ?? ""); + const updates = {}; + const generatedKeys = []; + + for (const [key, value] of Object.entries(DEFAULTS)) { + if (parsed[key] == null || String(parsed[key]).trim() === "") { + updates[key] = value; + generatedKeys.push(key); + } + } + + for (const key of SECRET_KEYS) { + if (isPlaceholderSecret(parsed[key])) { + if (!generateSecrets) continue; + updates[key] = randomSecret(); + generatedKeys.push(key); + } + } + + if ( + (parsed.AUTH_PUBLIC_BASE_URL == null || String(parsed.AUTH_PUBLIC_BASE_URL).trim() === "") && + updates.AUTH_PUBLIC_BASE_URL == null + ) { + const webPort = updates.WEB_PORT ?? parsed.WEB_PORT ?? DEFAULTS.WEB_PORT; + updates.AUTH_PUBLIC_BASE_URL = `http://127.0.0.1:${webPort}`; + generatedKeys.push("AUTH_PUBLIC_BASE_URL"); + } + + const text = Object.keys(updates).length > 0 + ? updateDeploymentEnvironment(sourceText ?? "", updates) + : sourceText?.endsWith("\n") || sourceText === "" + ? sourceText ?? "" + : `${sourceText}\n`; + + const env = { ...parseDeploymentEnvironment(text) }; + return { text, env, generatedKeys }; +} + +export function renderWebEnvironment(env) { + const authMode = env.DATAFOUNDRY_AUTH_MODE?.trim() || "password"; + const apiHost = env.API_HOST?.trim() || "127.0.0.1"; + const apiPort = env.API_PORT?.trim() || "8787"; + return [ + `NEXT_PUBLIC_DATAFOUNDRY_AUTH_MODE=${authMode}`, + "NEXT_PUBLIC_AGENT_RUNTIME_URL=", + "NEXT_PUBLIC_CONFIG_API_URL=", + `API_PROXY_TARGET=http://${apiHost}:${apiPort}`, + "" + ].join("\n"); +} + +async function writeAtomic(filePath, content, mode = 0o600) { + await mkdir(path.dirname(filePath), { recursive: true }); + const tempPath = path.join( + path.dirname(filePath), + `.${path.basename(filePath)}.${process.pid}.${Date.now()}.tmp` + ); + try { + await writeFile(tempPath, content, { encoding: "utf8", mode }); + await rename(tempPath, filePath); + } catch (error) { + await rm(tempPath, { force: true }).catch(() => {}); + throw error; + } +} + +export async function writeDeploymentConfiguration(root, rootText, webText, options = {}) { + const envPath = path.join(root, ".env"); + const webPath = path.join(root, "apps/web/.env.local"); + let backupPath; + + if (options.backup) { + const stamp = options.timestamp ?? new Date().toISOString().replace(/[-:TZ.]/g, "").slice(0, 14); + backupPath = path.join(root, `.env.backup-${stamp}`); + const { readFile } = await import("node:fs/promises"); + let existing = ""; + try { + existing = await readFile(envPath, "utf8"); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + await writeAtomic(backupPath, existing, 0o600); + } + + await writeAtomic(envPath, rootText, 0o600); + await writeAtomic(webPath, webText, 0o600); + return { envPath, webPath, backupPath }; +} + +export function redactSensitiveText(text) { + let result = String(text ?? ""); + + result = result.replace(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/gm, (full, key, value) => { + if (!SENSITIVE_KEY_PATTERN.test(key)) return full; + return `${key}=${maskSecret(value)}`; + }); + + result = result.replace( + /("([A-Za-z_][A-Za-z0-9_]*)"\s*:\s*")([^"]*)(")/g, + (full, prefix, key, value, suffix) => { + if (!SENSITIVE_KEY_PATTERN.test(key) && !SENSITIVE_JSON_KEY_PATTERN.test(key)) return full; + return `${prefix}${maskSecret(value)}${suffix}`; + } + ); + + result = result.replace( + /(Authorization:\s*Bearer\s+)(\S+)/gi, + (_, prefix) => `${prefix}${maskSecret("bearer-token")}` + ); + + result = result.replace( + /(https?:\/\/)([^/\s:@]+):([^/\s@]+)@/gi, + (_, protocol) => `${protocol}****:****@` + ); + + result = result.replace( + /\b((?:sk|rk|pk|tok)-[A-Za-z0-9_-]{8,}|(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{16,}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g, + (value) => maskSecret(value) + ); + + return result; +} diff --git a/scripts/deploy/config.test.mjs b/scripts/deploy/config.test.mjs new file mode 100644 index 00000000..7a7fcf01 --- /dev/null +++ b/scripts/deploy/config.test.mjs @@ -0,0 +1,130 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, stat, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { + ensureDeploymentEnvironment, + isCompleteDeploymentConfig, + parseDeploymentEnvironment, + redactSensitiveText, + renderWebEnvironment, + updateDeploymentEnvironment, + writeDeploymentConfiguration +} from "./config.mjs"; + +test("creates safe defaults without model settings", () => { + const result = ensureDeploymentEnvironment("", { randomSecret: () => "generated-secret-value" }); + assert.equal(result.env.WEB_PORT, "3000"); + assert.equal(result.env.API_PORT, "8787"); + assert.equal(result.env.AUTH_SESSION_SECRET, "generated-secret-value"); + assert.equal(result.env.SECRET_MASTER_KEY, "generated-secret-value"); + assert.equal(result.env.LLM_API_KEY, undefined); +}); + +test("preserves existing secrets and unrelated values", () => { + const source = "AUTH_SESSION_SECRET=existing-session\nSECRET_MASTER_KEY=existing-master\nCUSTOM_VALUE=keep-me\n"; + const result = ensureDeploymentEnvironment(source, { randomSecret: () => "replacement" }); + assert.match(result.text, /AUTH_SESSION_SECRET=existing-session/); + assert.match(result.text, /SECRET_MASTER_KEY=existing-master/); + assert.match(result.text, /CUSTOM_VALUE=keep-me/); +}); + +test("renders same-origin Web BFF configuration", () => { + const text = renderWebEnvironment({ + DATAFOUNDRY_AUTH_MODE: "password", + API_HOST: "127.0.0.1", + API_PORT: "8877" + }); + assert.match(text, /NEXT_PUBLIC_AGENT_RUNTIME_URL=$/m); + assert.match(text, /NEXT_PUBLIC_CONFIG_API_URL=$/m); + assert.match(text, /API_PROXY_TARGET=http:\/\/127\.0\.0\.1:8877/); +}); + +test("reconfigure creates a backup and atomically writes both files", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "datafoundry-config-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + await writeFile(path.join(root, ".env"), "AUTH_SESSION_SECRET=old\nSECRET_MASTER_KEY=old-master\n"); + const result = ensureDeploymentEnvironment(await readFile(path.join(root, ".env"), "utf8")); + const written = await writeDeploymentConfiguration(root, result.text, renderWebEnvironment(result.env), { + backup: true, + timestamp: "20260722-120000" + }); + assert.equal(await readFile(written.backupPath, "utf8"), "AUTH_SESSION_SECRET=old\nSECRET_MASTER_KEY=old-master\n"); + assert.match(await readFile(path.join(root, "apps/web/.env.local"), "utf8"), /API_PROXY_TARGET=/); + + if (process.platform !== "win32") { + for (const filePath of [ + path.join(root, ".env"), + path.join(root, "apps/web/.env.local"), + written.backupPath + ]) { + assert.equal((await stat(filePath)).mode & 0o777, 0o600, filePath); + } + } +}); + +test("updateDeploymentEnvironment upserts values while preserving comments", () => { + const source = "# comment\nWEB_PORT=3000\nCUSTOM=keep\n"; + const text = updateDeploymentEnvironment(source, { WEB_PORT: "3310", API_PORT: "8877" }); + assert.match(text, /# comment/); + assert.match(text, /WEB_PORT=3310/); + assert.match(text, /API_PORT=8877/); + assert.match(text, /CUSTOM=keep/); +}); + +test("redactSensitiveText masks secret-like keys", () => { + const redacted = redactSensitiveText("LLM_API_KEY=super-secret\nAUTH_SESSION_SECRET=abc\nWEB_PORT=3000\n"); + assert.match(redacted, /LLM_API_KEY=\*+/); + assert.match(redacted, /AUTH_SESSION_SECRET=\*+/); + assert.match(redacted, /WEB_PORT=3000/); + assert.doesNotMatch(redacted, /super-secret/); +}); + +test("redactSensitiveText masks JSON keys, bearer tokens, URL userinfo, and token prefixes", () => { + const redacted = redactSensitiveText( + [ + '{"apiKey":"json-secret-value","api_key":"also-secret"}', + "Authorization: Bearer fixture-deploy-secret-at-least-32-chars", + "https://user:fixture-deploy-secret-at-least-32-chars@example.com/path", + "token sk-abcdefghijklmnop", + "WEB_PORT=3000" + ].join("\n") + ); + assert.doesNotMatch(redacted, /json-secret-value|also-secret|fixture-deploy-secret-at-least-32-chars|sk-abcdefghijklmnop|user:fixture/); + assert.match(redacted, /Authorization: Bearer \*+/i); + assert.match(redacted, /https:\/\/\*\*\*\*:\*\*\*\*@example\.com\/path/); + assert.match(redacted, /WEB_PORT=3000/); +}); + +test("ensureDeploymentEnvironment can skip secret generation", () => { + const result = ensureDeploymentEnvironment("WEB_PORT=3000\n", { generateSecrets: false }); + assert.equal(result.env.AUTH_SESSION_SECRET, undefined); + assert.equal(result.env.SECRET_MASTER_KEY, undefined); + assert.ok(result.generatedKeys.includes("WEB_HOST")); + assert.ok(!result.generatedKeys.includes("AUTH_SESSION_SECRET")); +}); + +test("isCompleteDeploymentConfig rejects placeholders and partial env", () => { + assert.equal(isCompleteDeploymentConfig(parseDeploymentEnvironment("FOO=bar\n")), false); + assert.equal( + isCompleteDeploymentConfig({ + WEB_PORT: "3000", + API_PORT: "8787", + AUTH_PUBLIC_BASE_URL: "http://127.0.0.1:3000", + AUTH_SESSION_SECRET: "change-me", + SECRET_MASTER_KEY: "replace-me" + }), + false + ); + assert.equal( + isCompleteDeploymentConfig({ + WEB_PORT: "3000", + API_PORT: "8787", + AUTH_PUBLIC_BASE_URL: "http://127.0.0.1:3000", + AUTH_SESSION_SECRET: "existing-session-secret-value", + SECRET_MASTER_KEY: "existing-master-secret-value" + }), + true + ); +}); diff --git a/scripts/deploy/controller.mjs b/scripts/deploy/controller.mjs new file mode 100644 index 00000000..2927307c --- /dev/null +++ b/scripts/deploy/controller.mjs @@ -0,0 +1,157 @@ +import { deploymentHelp } from "./args.mjs"; + +export function createDeploymentController(deps) { + async function runDeploy(parsed) { + const calls = []; + let configurationChanged = false; + let serviceStopped = false; + let stage = "load-config"; + let config; + let logPath = "storage/logs/deploy-latest.log"; + + try { + if (deps.beginDeployLog) { + logPath = await deps.beginDeployLog(); + } + + stage = "load-config"; + config = await deps.loadConfiguration({ reconfigure: parsed.reconfigure }); + calls.push("load-config"); + + stage = "preflight"; + await deps.preflight(config, parsed); + calls.push("preflight"); + + stage = "configure"; + config = await deps.configure(config, parsed); + calls.push("configure"); + + stage = "check-dependencies"; + await deps.checkDependencies(config, parsed); + calls.push("check-dependencies"); + + stage = "select-ports"; + config = await deps.selectPorts(config, parsed); + calls.push("select-ports"); + + stage = "write-config"; + await deps.writeConfiguration(config, parsed); + configurationChanged = true; + calls.push("write-config"); + + const running = await deps.isRunning(); + if (running) { + stage = "stop-old"; + await deps.stop(); + serviceStopped = true; + calls.push("stop-old"); + } + + stage = "install"; + await deps.installProject(config, parsed); + calls.push("install"); + + stage = "build-typescript"; + await deps.buildTypeScript(config, parsed); + calls.push("build-typescript"); + + stage = "build-web"; + await deps.buildWeb(config, parsed); + calls.push("build-web"); + + stage = "build-tui"; + await deps.buildTui(config, parsed); + calls.push("build-tui"); + + stage = "verify-ports-again"; + await deps.verifyPorts(config, parsed); + calls.push("verify-ports-again"); + + stage = "start"; + await deps.start(config, parsed); + calls.push("start"); + + stage = "wait-for-health"; + await deps.waitForHealth(config, parsed); + calls.push("wait-for-health"); + + stage = "mark-healthy"; + await deps.markHealthy(config, parsed); + calls.push("mark-healthy"); + + return { ok: true, calls, configurationChanged, serviceStopped }; + } catch (error) { + const failure = { + stage, + summary: error?.message ?? String(error), + error, + configurationChanged, + serviceStopped, + maintenanceWindow: serviceStopped, + oldServiceRunning: false, + logPath: error?.logPath ?? logPath, + retryCommand: "./deploy.sh deploy", + doctorCommand: "./deploy.sh doctor", + calls + }; + await deps.reportFailure(failure); + const wrapped = new Error(error?.message ?? String(error)); + wrapped.failure = failure; + throw wrapped; + } + } + + async function runStart() { + const config = await deps.loadConfiguration({ reconfigure: false }); + await deps.preflight(config, { command: "start" }); + await deps.start(config, { command: "start" }); + await deps.waitForHealth(config, { command: "start" }); + await deps.markHealthy(config, { command: "start" }); + return { ok: true }; + } + + async function runStop() { + await deps.stop(); + return { ok: true }; + } + + async function runRestart() { + await deps.stop(); + return runStart(); + } + + async function runTui(parsed) { + const config = await deps.loadConfiguration({ reconfigure: false }); + await deps.ensureTuiReady(config, parsed); + await deps.checkApiForTui(config, parsed); + await deps.startTui(config, parsed); + return { ok: true }; + } + + return { + async run(parsed) { + switch (parsed.command) { + case "deploy": + return runDeploy(parsed); + case "start": + return runStart(); + case "stop": + return runStop(); + case "restart": + return runRestart(); + case "status": + return deps.status(); + case "logs": + return deps.logs(); + case "doctor": + return deps.doctor(); + case "tui": + return runTui(parsed); + case "help": + return deps.printHelp(deploymentHelp()); + default: + throw new Error(`Unknown command: ${parsed.command}`); + } + } + }; +} diff --git a/scripts/deploy/controller.test.mjs b/scripts/deploy/controller.test.mjs new file mode 100644 index 00000000..9e5ef24b --- /dev/null +++ b/scripts/deploy/controller.test.mjs @@ -0,0 +1,170 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createDeploymentController } from "./controller.mjs"; + +test("deploy update path uses exact maintenance-window ordering", async () => { + const controller = createDeploymentController({ + async loadConfiguration() { + return { env: {} }; + }, + async preflight() {}, + async configure(config) { + return config; + }, + async checkDependencies() {}, + async selectPorts(config) { + return config; + }, + async writeConfiguration() {}, + async isRunning() { + return true; + }, + async stop() {}, + async installProject() {}, + async buildTypeScript() {}, + async buildWeb() {}, + async buildTui() {}, + async verifyPorts() {}, + async start() {}, + async waitForHealth() {}, + async markHealthy() {}, + async reportFailure() {} + }); + + const result = await controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }); + assert.deepEqual(result.calls, [ + "load-config", + "preflight", + "configure", + "check-dependencies", + "select-ports", + "write-config", + "stop-old", + "install", + "build-typescript", + "build-web", + "build-tui", + "verify-ports-again", + "start", + "wait-for-health", + "mark-healthy" + ]); + assert.ok(!result.calls.includes("start-tui")); +}); + +test("build failure after stop never starts and reports maintenance window", async () => { + let failure; + const controller = createDeploymentController({ + async loadConfiguration() { + return { env: {} }; + }, + async preflight() {}, + async configure(config) { + return config; + }, + async checkDependencies() {}, + async selectPorts(config) { + return config; + }, + async writeConfiguration() {}, + async isRunning() { + return true; + }, + async stop() {}, + async installProject() {}, + async buildTypeScript() { + const error = new Error("build failed"); + error.logPath = "storage/logs/deploy-20260722.log"; + throw error; + }, + async buildWeb() {}, + async buildTui() {}, + async verifyPorts() {}, + async start() {}, + async waitForHealth() {}, + async markHealthy() {}, + async reportFailure(details) { + failure = details; + } + }); + + const storage = { sentinel: Buffer.from("keep-me") }; + await assert.rejects( + () => controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }), + /build failed/ + ); + assert.ok(!failure.calls.includes("start")); + assert.ok(failure.calls.includes("stop-old")); + assert.equal(failure.maintenanceWindow, true); + assert.equal(failure.serviceStopped, true); + assert.equal(failure.stage, "build-typescript"); + assert.equal(storage.sentinel.toString(), "keep-me"); +}); + +test("stop and help dispatch", async () => { + const calls = []; + const controller = createDeploymentController({ + async stop() { + calls.push("stop"); + }, + async printHelp() { + calls.push("help"); + } + }); + await controller.run({ command: "stop" }); + await controller.run({ command: "help" }); + assert.deepEqual(calls, ["stop", "help"]); +}); + +test("optional tui command checks readiness and API then starts foreground client", async () => { + const calls = []; + const controller = createDeploymentController({ + async loadConfiguration() { + calls.push("load-config"); + return { env: { API_PORT: "8787" } }; + }, + async ensureTuiReady() { + calls.push("ensure-tui-ready"); + }, + async checkApiForTui() { + calls.push("check-api-for-tui"); + }, + async startTui(_config, parsed) { + calls.push(`start-tui:${parsed.runtimeUrl ?? "default"}`); + } + }); + await controller.run({ + command: "tui", + reconfigure: false, + nonInteractive: false, + runtimeUrl: "http://127.0.0.1:9000/api/copilotkit" + }); + assert.deepEqual(calls, [ + "load-config", + "ensure-tui-ready", + "check-api-for-tui", + "start-tui:http://127.0.0.1:9000/api/copilotkit" + ]); +}); + +test("tui aborts before start when API check fails", async () => { + const calls = []; + const controller = createDeploymentController({ + async loadConfiguration() { + return { env: { API_PORT: "8787" } }; + }, + async ensureTuiReady() { + calls.push("ensure-tui-ready"); + }, + async checkApiForTui() { + calls.push("check-api-for-tui"); + throw new Error("API is not healthy"); + }, + async startTui() { + calls.push("start-tui"); + } + }); + await assert.rejects(() => controller.run({ command: "tui", runtimeUrl: null }), /API is not healthy/); + assert.deepEqual(calls, ["ensure-tui-ready", "check-api-for-tui"]); + assert.ok(!calls.includes("start-tui")); +}); diff --git a/scripts/deploy/dependencies.mjs b/scripts/deploy/dependencies.mjs new file mode 100644 index 00000000..c9820588 --- /dev/null +++ b/scripts/deploy/dependencies.mjs @@ -0,0 +1,133 @@ +import { spawn } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), "../.."); + +async function defaultRun(command, args = [], options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: options.cwd ?? ROOT, + env: options.env ?? process.env, + stdio: ["ignore", "pipe", "pipe"] + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) resolve({ stdout, stderr, code }); + else { + const error = new Error(stderr.trim() || `${command} exited ${code}`); + error.code = code; + error.stdout = stdout; + error.stderr = stderr; + reject(error); + } + }); + }); +} + +function parseNodeVersion(text) { + const match = /v?(\d+)\.(\d+)\.(\d+)/.exec(text ?? ""); + if (!match) return null; + return { major: Number(match[1]), minor: Number(match[2]), patch: Number(match[3]), raw: match[0] }; +} + +async function safeRun(run, command, args = []) { + try { + return await run(command, args); + } catch (error) { + return { error, stdout: error?.stdout ?? "", stderr: error?.stderr ?? "" }; + } +} + +export async function inspectDependencies(options = {}) { + const run = options.run ?? defaultRun; + const entries = []; + + const node = await safeRun(run, "node", ["--version"]); + const nodeVersion = parseNodeVersion(node.stdout || node.stderr); + entries.push({ + name: "node", + required: true, + foundVersion: nodeVersion?.raw ?? null, + minimumVersion: "22", + status: nodeVersion && nodeVersion.major >= 22 ? "ok" : "missing", + installAction: "node" + }); + + const npm = await safeRun(run, "npm", ["--version"]); + const npmVersion = (npm.stdout || "").trim() || null; + entries.push({ + name: "npm", + required: true, + foundVersion: npmVersion, + minimumVersion: "any", + status: npmVersion ? "ok" : "missing", + installAction: "node" + }); + + return entries; +} + +async function canInstallNonInteractive(options, run) { + if ((options.uid ?? process.getuid?.() ?? 1000) === 0) return true; + const result = await safeRun(run, "sudo", ["-n", "true"]); + return !result.error; +} + +export async function ensureDependencies(options = {}) { + const run = options.run ?? defaultRun; + const ask = options.ask; + const install = options.install ?? (async (action, installOptions = {}) => { + const args = [path.join(ROOT, "scripts/deploy/install-dependency.sh"), action]; + if (installOptions.nonInteractive) { + args.push("--non-interactive"); + } + await run("bash", args); + }); + const print = options.print ?? ((message) => process.stdout.write(`${message}\n`)); + + let entries = await inspectDependencies(options); + const missing = entries.filter((entry) => entry.status !== "ok"); + if (missing.length === 0) return entries; + + for (const entry of missing) { + const action = entry.installAction; + const commandHint = `bash scripts/deploy/install-dependency.sh ${action}`; + if (options.nonInteractive) { + const allowed = await canInstallNonInteractive(options, run); + if (!allowed) { + throw new Error( + `Missing ${entry.name}. Non-interactive install requires root or passwordless sudo. Or run: ${commandHint}` + ); + } + print(`Installing ${entry.name} via ${commandHint}`); + await install(action, { nonInteractive: true }); + continue; + } + + print(`缺少依赖：${entry.name}（需要 ${entry.minimumVersion}，当前 ${entry.foundVersion ?? "未找到"}）`); + print(`将执行：${commandHint}`); + const answer = String(await ask("是否安装？[y/N]：" )).trim().toLowerCase(); + if (answer !== "y" && answer !== "yes") { + throw new Error(`Dependency ${entry.name} is required. Install with: ${commandHint}`); + } + await install(action, { nonInteractive: false }); + } + + entries = await inspectDependencies(options); + const stillMissing = entries.filter((entry) => entry.status !== "ok"); + if (stillMissing.length > 0) { + throw new Error( + `Dependencies still missing after install: ${stillMissing.map((e) => e.name).join(", ")}` + ); + } + return entries; +} diff --git a/scripts/deploy/dependencies.test.mjs b/scripts/deploy/dependencies.test.mjs new file mode 100644 index 00000000..aa20fd2e --- /dev/null +++ b/scripts/deploy/dependencies.test.mjs @@ -0,0 +1,88 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { ensureDependencies, inspectDependencies } from "./dependencies.mjs"; + +function runner(map) { + return async (command, args = []) => { + const key = [command, ...args].join(" "); + if (!(key in map) && !(command in map)) { + const error = new Error(`missing mock for ${key}`); + error.code = "ENOENT"; + throw error; + } + const value = map[key] ?? map[command]; + if (typeof value === "function") return value(args); + if (value instanceof Error) throw value; + return value; + }; +} + +test("requires Node 22 and npm only", async () => { + const entries = await inspectDependencies({ + run: runner({ + "node --version": { stdout: "v22.14.0\n" }, + "npm --version": { stdout: "10.9.0\n" } + }) + }); + assert.equal(entries.find((e) => e.name === "node").status, "ok"); + assert.equal(entries.find((e) => e.name === "npm").status, "ok"); + assert.equal(entries.find((e) => e.name === "python"), undefined); + assert.equal(entries.find((e) => e.name === "uv"), undefined); +}); + +test("interactive refusal returns corrective install command", async () => { + await assert.rejects( + () => ensureDependencies({ + nonInteractive: false, + ask: async () => "n", + run: runner({ + "node --version": { stdout: "v20.0.0\n" }, + "npm --version": { stdout: "10.0.0\n" } + }), + install: async () => assert.fail("must not install") + }), + /install-dependency\.sh node|Node\.js 22/ + ); +}); + +test("non-interactive install requires root or passwordless sudo", async () => { + await assert.rejects( + () => ensureDependencies({ + nonInteractive: true, + uid: 1000, + ask: async () => assert.fail("must not prompt"), + run: runner({ + "node --version": { stdout: "v20.0.0\n" }, + "npm --version": { stdout: "10.0.0\n" }, + "sudo -n true": Object.assign(new Error("sudo failed"), { code: 1 }) + }), + install: async () => assert.fail("must not install") + }), + /root|passwordless sudo|non-interactive/i + ); +}); + +test("non-interactive install passes --non-interactive to install-dependency.sh", async () => { + const installs = []; + let nodeCalls = 0; + await ensureDependencies({ + nonInteractive: true, + uid: 0, + ask: async () => assert.fail("must not prompt"), + run: async (command, args = []) => { + const key = [command, ...args].join(" "); + if (key === "node --version") { + nodeCalls += 1; + return nodeCalls === 1 ? { stdout: "v20.0.0\n" } : { stdout: "v22.14.0\n" }; + } + if (key === "npm --version") return { stdout: "10.9.0\n" }; + throw new Error(`missing mock for ${key}`); + }, + install: async (action, installOptions) => { + installs.push({ action, installOptions }); + } + }); + assert.equal(installs.length, 1); + assert.equal(installs[0].action, "node"); + assert.equal(installs[0].installOptions.nonInteractive, true); +}); diff --git a/scripts/deploy/health.mjs b/scripts/deploy/health.mjs new file mode 100644 index 00000000..0fde3108 --- /dev/null +++ b/scripts/deploy/health.mjs @@ -0,0 +1,92 @@ +import net from "node:net"; + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export async function probeHttp(url, options = {}) { + const timeoutMs = options.timeoutMs ?? 5000; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { signal: controller.signal, redirect: "manual" }); + return { ok: response.status >= 200 && response.status < 400, status: response.status }; + } catch (error) { + return { ok: false, status: null, error: error?.message ?? String(error) }; + } finally { + clearTimeout(timer); + } +} + +export async function probeTcp(host, port, options = {}) { + const timeoutMs = options.timeoutMs ?? 5000; + return new Promise((resolve) => { + const socket = net.connect({ host, port }); + const timer = setTimeout(() => { + socket.destroy(); + resolve({ ok: false, error: "timeout" }); + }, timeoutMs); + socket.once("connect", () => { + clearTimeout(timer); + socket.end(); + resolve({ ok: true }); + }); + socket.once("error", (error) => { + clearTimeout(timer); + resolve({ ok: false, error: error.message }); + }); + }); +} + +async function resolveProcessAlive(config) { + if (typeof config.checkProcessAlive === "function") { + return Boolean(await config.checkProcessAlive()); + } + return Boolean(config.processAlive); +} + +export async function collectDeploymentHealth(config, options = {}) { + const probeTimeoutMs = options.probeTimeoutMs ?? 5000; + const processAlive = await resolveProcessAlive(config); + const processStatus = processAlive ? "running" : "stopped"; + + const health = await probeHttp(`${config.apiBaseUrl}/healthz`, { timeoutMs: probeTimeoutMs }); + const ready = await probeHttp(`${config.apiBaseUrl}/ready`, { timeoutMs: probeTimeoutMs }); + const web = await probeHttp(config.webUrl, { timeoutMs: probeTimeoutMs }); + + const apiHealth = health.ok && health.status === 200 ? "healthy" : "unhealthy"; + const apiReady = ready.ok && ready.status === 200 ? "ready" : "unhealthy"; + const webStatus = web.ok ? "reachable" : "unreachable"; + const ok = + processStatus === "running" && + apiHealth === "healthy" && + apiReady === "ready" && + webStatus === "reachable"; + + return { + process: processStatus, + apiHealth, + apiReady, + web: webStatus, + ok + }; +} + +export async function waitForDeployment(config, options = {}) { + const intervalMs = options.intervalMs ?? 1000; + const timeoutMs = options.timeoutMs ?? 60_000; + const deadline = Date.now() + timeoutMs; + let summary; + + while (Date.now() <= deadline) { + summary = await collectDeploymentHealth(config, options); + if (summary.ok) return summary; + await sleep(intervalMs); + } + + const error = new Error( + `Deployment health check timed out: process=${summary.process} apiHealth=${summary.apiHealth} apiReady=${summary.apiReady} web=${summary.web}` + ); + error.summary = summary; + throw error; +} diff --git a/scripts/deploy/health.test.mjs b/scripts/deploy/health.test.mjs new file mode 100644 index 00000000..8ebb1b02 --- /dev/null +++ b/scripts/deploy/health.test.mjs @@ -0,0 +1,174 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import net from "node:net"; +import test from "node:test"; +import { + collectDeploymentHealth, + probeHttp, + probeTcp, + waitForDeployment +} from "./health.mjs"; + +function listen(server) { + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => resolve(server.address().port)); + }); +} + +test("probeHttp and probeTcp succeed against ephemeral servers", async () => { + const server = http.createServer((_req, res) => { + res.writeHead(200); + res.end("ok"); + }); + const port = await listen(server); + try { + const result = await probeHttp(`http://127.0.0.1:${port}/`); + assert.equal(result.ok, true); + assert.equal(result.status, 200); + const tcp = await probeTcp("127.0.0.1", port); + assert.equal(tcp.ok, true); + } finally { + server.close(); + } +}); + +test("collectDeploymentHealth reports healthy stack", async () => { + const api = http.createServer((req, res) => { + if (req.url === "/healthz" || req.url === "/ready") { + res.writeHead(200); + res.end("ok"); + return; + } + res.writeHead(404); + res.end(); + }); + const web = http.createServer((_req, res) => { + res.writeHead(302); + res.end(); + }); + const apiPort = await listen(api); + const webPort = await listen(web); + try { + const summary = await collectDeploymentHealth({ + processAlive: true, + apiBaseUrl: `http://127.0.0.1:${apiPort}`, + webUrl: `http://127.0.0.1:${webPort}` + }); + assert.deepEqual(summary, { + process: "running", + apiHealth: "healthy", + apiReady: "ready", + web: "reachable", + ok: true + }); + } finally { + api.close(); + web.close(); + } +}); + +test("waitForDeployment retries until ready then times out with summary", async () => { + let ready = false; + const api = http.createServer((req, res) => { + if (req.url === "/healthz") { + res.writeHead(200); + res.end("ok"); + return; + } + if (req.url === "/ready") { + res.writeHead(ready ? 200 : 503); + res.end(ready ? "ok" : "starting"); + return; + } + res.writeHead(404); + res.end(); + }); + const web = http.createServer((_req, res) => { + res.writeHead(200); + res.end("web"); + }); + const apiPort = await listen(api); + const webPort = await listen(web); + const config = { + processAlive: true, + apiBaseUrl: `http://127.0.0.1:${apiPort}`, + webUrl: `http://127.0.0.1:${webPort}` + }; + + try { + setTimeout(() => { + ready = true; + }, 80); + const summary = await waitForDeployment(config, { + intervalMs: 40, + timeoutMs: 1000, + probeTimeoutMs: 200 + }); + assert.equal(summary.ok, true); + + await assert.rejects( + () => waitForDeployment({ ...config, processAlive: false }, { + intervalMs: 20, + timeoutMs: 60, + probeTimeoutMs: 50 + }), + /process|unhealthy|timeout/i + ); + } finally { + api.close(); + web.close(); + } +}); + +test("waitForDeployment rechecks processAlive each round via callback", async () => { + let aliveChecks = 0; + // Keep HTTP healthy so the only failing gate is processAlive, avoiding a race + // where /ready flips true while the callback still reports alive. + const api = http.createServer((req, res) => { + if (req.url === "/healthz" || req.url === "/ready") { + res.writeHead(200); + res.end("ok"); + return; + } + res.writeHead(404); + res.end("missing"); + }); + const web = http.createServer((_req, res) => { + res.writeHead(200); + res.end("web"); + }); + const apiPort = await listen(api); + const webPort = await listen(web); + + try { + await assert.rejects( + () => + waitForDeployment( + { + checkProcessAlive: async () => { + aliveChecks += 1; + return false; + }, + apiBaseUrl: `http://127.0.0.1:${apiPort}`, + webUrl: `http://127.0.0.1:${webPort}` + }, + { intervalMs: 20, timeoutMs: 120, probeTimeoutMs: 50 } + ), + /process=stopped|timed out/i + ); + assert.ok(aliveChecks >= 2); + } finally { + api.close(); + web.close(); + } +}); + +test("TCP probe fails for closed ports", async () => { + const server = net.createServer(); + const port = await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => resolve(server.address().port)); + }); + await new Promise((resolve) => server.close(resolve)); + const result = await probeTcp("127.0.0.1", port, { timeoutMs: 100 }); + assert.equal(result.ok, false); +}); diff --git a/scripts/deploy/install-dependency.sh b/scripts/deploy/install-dependency.sh new file mode 100755 index 00000000..88dc5bff --- /dev/null +++ b/scripts/deploy/install-dependency.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ACTION="${1:-}" +NON_INTERACTIVE=0 +if [[ "${2:-}" == "--non-interactive" ]] || [[ "${DEPLOY_NON_INTERACTIVE:-}" == "1" ]]; then + NON_INTERACTIVE=1 +fi + +usage() { + echo "Usage: install-dependency.sh [--non-interactive]" >&2 + exit 2 +} + +case "${ACTION}" in + node) ;; + *) usage ;; +esac + +run_privileged() { + if [[ "$(id -u)" -eq 0 ]]; then + "$@" + elif [[ "${NON_INTERACTIVE}" -eq 1 ]]; then + sudo -n "$@" + else + sudo "$@" + fi +} + +install_node() { + local setup + local nodesource_url="https://deb.nodesource.com/setup_22.x" + setup="$(mktemp)" + trap 'rm -f "${setup}"' RETURN + curl -fsSL "${nodesource_url}" -o "${setup}" + run_privileged bash "${setup}" + run_privileged apt-get install -y nodejs + rm -f "${setup}" + trap - RETURN + node --version >/dev/null + npm --version >/dev/null +} + +case "${ACTION}" in + node) install_node ;; +esac diff --git a/scripts/deploy/lifecycle.test.mjs b/scripts/deploy/lifecycle.test.mjs new file mode 100644 index 00000000..6419d38b --- /dev/null +++ b/scripts/deploy/lifecycle.test.mjs @@ -0,0 +1,335 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createDeploymentController } from "./controller.mjs"; +import { redactSensitiveText } from "./config.mjs"; +import { formatDeploymentFailure } from "./cli.mjs"; + +function createFakeDeployDeps(overrides = {}) { + const calls = []; + const storage = { sentinel: Buffer.from("storage-sentinel-bytes") }; + let running = Boolean(overrides.running); + let statusState = { process: "stopped", apiHealth: "unhealthy", apiReady: "unhealthy", web: "unreachable", ok: false }; + + const deps = { + calls, + storage, + async loadConfiguration() { + calls.push("load-config"); + return { env: { WEB_PORT: "3000", API_PORT: "8787" }, envText: "", webText: "" }; + }, + async preflight(_config, options) { + calls.push("preflight"); + if (options?.command === "start" && overrides.missingBuild) { + throw new Error("请先运行 ./deploy.sh deploy"); + } + }, + async configure(config) { + calls.push("configure"); + return config; + }, + async checkDependencies() { + calls.push("check-dependencies"); + }, + async selectPorts(config) { + calls.push("select-ports"); + return config; + }, + async writeConfiguration() { + calls.push("write-config"); + }, + async isRunning() { + return running; + }, + async stop() { + calls.push("stop"); + running = false; + statusState = { ...statusState, process: "stopped", ok: false }; + }, + async installProject() { + calls.push("install"); + }, + async buildTypeScript() { + calls.push("build-typescript"); + if (overrides.failBuild) { + const error = new Error("TypeScript build failed"); + error.logPath = "storage/logs/deploy-20260722-143000.log"; + throw error; + } + }, + async buildWeb() { + calls.push("build-web"); + if (overrides.failWebBuild) { + const error = new Error("Web build failed"); + error.logPath = "storage/logs/deploy-20260722-143000.log"; + throw error; + } + }, + async buildTui() { + calls.push("build-tui"); + if (overrides.failTuiBuild) { + const error = new Error("TUI build failed"); + error.logPath = "storage/logs/deploy-20260722-143000.log"; + throw error; + } + }, + async ensureTuiReady() { + calls.push("ensure-tui-ready"); + }, + async checkApiForTui() { + calls.push("check-api-for-tui"); + if (overrides.tuiApiUnhealthy) { + throw new Error("API is not healthy at http://127.0.0.1:8787/healthz"); + } + }, + async startTui() { + calls.push("start-tui"); + }, + + async verifyPorts() { + calls.push("verify-ports-again"); + if (overrides.portConflictAfterBuild) { + throw new Error("Web port 3000 is already in use"); + } + }, + async start() { + calls.push("start"); + running = true; + statusState = { + process: "running", + apiHealth: "healthy", + apiReady: "ready", + web: "reachable", + ok: true + }; + }, + async waitForHealth() { + calls.push("wait-for-health"); + }, + async markHealthy() { + calls.push("mark-healthy"); + }, + async status() { + calls.push("status"); + return statusState; + }, + async logs() { + calls.push("logs"); + }, + async doctor() { + calls.push("doctor"); + }, + async printHelp() { + calls.push("help"); + }, + async reportFailure(failure) { + calls.push("report-failure"); + deps.lastFailure = failure; + }, + ...overrides.deps + }; + return deps; +} + +test("fresh deploy runs configure through healthy state", async () => { + const deps = createFakeDeployDeps({ running: false }); + const controller = createDeploymentController(deps); + const result = await controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }); + assert.ok(result.ok); + assert.deepEqual(result.calls, [ + "load-config", + "preflight", + "configure", + "check-dependencies", + "select-ports", + "write-config", + "install", + "build-typescript", + "build-web", + "build-tui", + "verify-ports-again", + "start", + "wait-for-health", + "mark-healthy" + ]); + assert.ok(!result.calls.includes("stop-old")); + assert.ok(!result.calls.includes("start-tui")); +}); + +test("update deploy stops only after preflight and before install/build", async () => { + const deps = createFakeDeployDeps({ running: true }); + const controller = createDeploymentController(deps); + const result = await controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }); + const stopIndex = result.calls.indexOf("stop-old"); + const installIndex = result.calls.indexOf("install"); + assert.ok(stopIndex > result.calls.indexOf("preflight")); + assert.ok(stopIndex > result.calls.indexOf("write-config")); + assert.ok(installIndex > stopIndex); + assert.ok(result.calls.indexOf("build-typescript") > stopIndex); +}); + +test("build failure keeps storage intact and reports maintenance window", async () => { + const deps = createFakeDeployDeps({ running: true, failWebBuild: true }); + const controller = createDeploymentController(deps); + await assert.rejects( + () => controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }), + /Web build failed/ + ); + assert.equal(deps.storage.sentinel.toString(), "storage-sentinel-bytes"); + assert.equal(deps.lastFailure.maintenanceWindow, true); + assert.equal(deps.lastFailure.oldServiceRunning, false); + assert.equal(deps.lastFailure.configurationChanged, true); + assert.equal(deps.lastFailure.stage, "build-web"); + assert.equal(deps.lastFailure.retryCommand, "./deploy.sh deploy"); + assert.equal(deps.lastFailure.doctorCommand, "./deploy.sh doctor"); + assert.ok(!deps.calls.includes("start")); + const formatted = formatDeploymentFailure({ + ...deps.lastFailure, + summary: "Web build failed" + }); + assert.match(formatted, /现有数据未被修改/); + assert.match(formatted, /维护窗口/); + assert.doesNotMatch(formatted, /已恢复|rolled back|restored/i); +}); + +test("port conflict after build prevents start and never kills listeners", async () => { + const deps = createFakeDeployDeps({ running: false, portConflictAfterBuild: true }); + let killed = false; + deps.killListener = () => { + killed = true; + }; + const controller = createDeploymentController(deps); + await assert.rejects( + () => controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }), + /already in use/ + ); + assert.ok(!deps.calls.includes("start")); + assert.equal(killed, false); +}); + +test("start with missing build artifact fails with deploy hint", async () => { + const deps = createFakeDeployDeps({ missingBuild: true }); + const controller = createDeploymentController(deps); + await assert.rejects(() => controller.run({ command: "start" }), /请先运行 \.\/deploy\.sh deploy/); + assert.ok(!deps.calls.includes("install")); + assert.ok(!deps.calls.includes("build-typescript")); +}); + +test("start with incomplete config fails with deploy hint and never starts", async () => { + const deps = createFakeDeployDeps(); + deps.preflight = async (_config, options) => { + deps.calls.push("preflight"); + if (options?.command === "start") { + throw new Error("缺少合法完整的 .env 配置，请先运行 ./deploy.sh deploy"); + } + }; + const controller = createDeploymentController(deps); + await assert.rejects(() => controller.run({ command: "start" }), /缺少合法完整的 \.env|请先运行 \.\/deploy\.sh deploy/); + assert.ok(!deps.calls.includes("start")); +}); + +test("status reports stopped and unhealthy components", async () => { + const deps = createFakeDeployDeps(); + deps.status = async () => ({ + process: "stopped", + apiHealth: "unhealthy", + apiReady: "unhealthy", + web: "unreachable", + ok: false + }); + const controller = createDeploymentController(deps); + const summary = await controller.run({ command: "status" }); + assert.equal(summary.process, "stopped"); + assert.equal(summary.apiHealth, "unhealthy"); + assert.equal(summary.web, "unreachable"); +}); + +test("restart does not install or build", async () => { + const deps = createFakeDeployDeps({ running: true }); + const controller = createDeploymentController(deps); + await controller.run({ command: "restart" }); + assert.ok(deps.calls.includes("stop")); + assert.ok(deps.calls.includes("start")); + assert.ok(!deps.calls.includes("install")); + assert.ok(!deps.calls.includes("build-typescript")); + assert.ok(!deps.calls.includes("build-web")); +}); + +test("stop twice succeeds both times", async () => { + const deps = createFakeDeployDeps({ running: true }); + const controller = createDeploymentController(deps); + await controller.run({ command: "stop" }); + await controller.run({ command: "stop" }); + assert.equal(deps.calls.filter((call) => call === "stop").length, 2); +}); + +test("runtime and deploy logs redact fixture secrets", () => { + const text = redactSensitiveText( + "AUTH_SESSION_SECRET=fixture-deploy-secret-at-least-32-chars\nLLM_API_KEY=sk-test\nWEB_PORT=3000\n" + ); + assert.doesNotMatch(text, /fixture-deploy-secret-at-least-32-chars/); + assert.doesNotMatch(text, /sk-test/); + assert.match(text, /WEB_PORT=3000/); +}); + +test("deploy failure uses beginDeployLog path", async () => { + let failure; + const controller = createDeploymentController({ + async beginDeployLog() { + return "storage/logs/deploy-20260722143000.log"; + }, + async loadConfiguration() { + return { env: {} }; + }, + async preflight() {}, + async configure(config) { + return config; + }, + async checkDependencies() {}, + async selectPorts(config) { + return config; + }, + async writeConfiguration() {}, + async isRunning() { + return false; + }, + async stop() {}, + async installProject() {}, + async buildTypeScript() { + throw new Error("TypeScript build failed"); + }, + async buildWeb() {}, + async buildTui() {}, + async verifyPorts() {}, + async start() {}, + async waitForHealth() {}, + async markHealthy() {}, + async reportFailure(details) { + failure = details; + } + }); + + await assert.rejects( + () => controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }), + /TypeScript build failed/ + ); + assert.equal(failure.logPath, "storage/logs/deploy-20260722143000.log"); +}); + +test("deploy never auto-starts TUI client", async () => { + const deps = createFakeDeployDeps({ running: false }); + const controller = createDeploymentController(deps); + const result = await controller.run({ command: "deploy", reconfigure: false, nonInteractive: true }); + assert.ok(result.calls.includes("build-tui")); + assert.ok(!result.calls.includes("ensure-tui-ready")); + assert.ok(!result.calls.includes("check-api-for-tui")); + assert.ok(!result.calls.includes("start-tui")); +}); + +test("optional tui command requires healthy API before start", async () => { + const deps = createFakeDeployDeps({ tuiApiUnhealthy: true }); + const controller = createDeploymentController(deps); + await assert.rejects(() => controller.run({ command: "tui", runtimeUrl: null }), /API is not healthy/); + assert.ok(deps.calls.includes("ensure-tui-ready")); + assert.ok(deps.calls.includes("check-api-for-tui")); + assert.ok(!deps.calls.includes("start-tui")); +}); diff --git a/scripts/deploy/ports.mjs b/scripts/deploy/ports.mjs new file mode 100644 index 00000000..f36b840b --- /dev/null +++ b/scripts/deploy/ports.mjs @@ -0,0 +1,127 @@ +import { execFile } from "node:child_process"; +import net from "node:net"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +export function parsePort(value, label) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1 || parsed > 65535) { + throw new Error(`${label} port must be an integer between 1 and 65535`); + } + return parsed; +} + +export async function describePortOwner(port) { + if (process.platform !== "linux") return "unknown process"; + try { + const { stdout } = await execFileAsync("ss", ["-ltnp"], { + encoding: "utf8", + maxBuffer: 2 * 1024 * 1024 + }); + const needle = `:${port}`; + for (const line of stdout.split(/\r?\n/u)) { + if (!line.includes(needle) || !/\bLISTEN\b/u.test(line)) continue; + const users = /users:\(\("([^"]+)",pid=(\d+)/u.exec(line); + if (users) return `${users[1]} pid=${users[2]}`; + return "unknown process"; + } + } catch { + // best effort + } + return "unknown process"; +} + +export async function probePort(host, port, options = {}) { + const describe = options.describeOwner ?? describePortOwner; + const available = await new Promise((resolve) => { + const server = net.createServer(); + server.once("error", () => resolve(false)); + server.listen({ host, port, exclusive: true }, () => { + server.close(() => resolve(true)); + }); + }); + if (available) return { available: true, owner: null }; + return { available: false, owner: await describe(port) }; +} + +function defaultPrint(message) { + process.stdout.write(`${message}\n`); +} + +export async function selectDeploymentPort(options) { + const { + label, + defaultPort, + reserved = new Set(), + managedPorts = new Set(), + nonInteractive = false, + ask, + probe = (port) => probePort("0.0.0.0", port), + print = defaultPrint + } = options; + + let candidate = parsePort(defaultPort, label); + let explicitChoice = false; + + while (true) { + if (reserved.has(candidate)) { + const message = `${label} port ${candidate} is already selected for another service`; + if (nonInteractive) throw new Error(message); + print(message); + const answer = String(await ask(`请为 ${label} 输入其他端口，或输入 q 退出：`)).trim(); + if (answer.toLowerCase() === "q") throw new Error("port selection cancelled"); + candidate = parsePort(answer, label); + explicitChoice = true; + continue; + } + + const result = await probe(candidate); + const managed = managedPorts.has(candidate); + + if (result.available || managed) { + if (nonInteractive || explicitChoice) return candidate; + + print(`端口 ${candidate} 当前可用，请选择：`); + print(`1. 使用端口 ${candidate}`); + print("2. 指定其他端口"); + const choice = String((await ask("请选择 [1]：")) ?? "").trim() || "1"; + if (choice === "1") return candidate; + if (choice === "2") { + const answer = String(await ask(`请输入 ${label} 端口：`)).trim(); + candidate = parsePort(answer, label); + explicitChoice = true; + continue; + } + print("请输入 1 或 2"); + continue; + } + + if (nonInteractive) { + throw new Error(`${label} port ${candidate} is already in use`); + } + + const owner = result.owner || "unknown process"; + print(`端口 ${candidate} 已被未知进程占用（${owner}）。DataFoundry 不会结束该进程。`); + const answer = String(await ask("请输入其他端口，或输入 q 退出：")).trim(); + if (answer.toLowerCase() === "q") throw new Error("port selection cancelled"); + candidate = parsePort(answer, label); + explicitChoice = true; + } +} + +export async function verifySelectedPorts(services, options = {}) { + const { + managedPorts = new Set(), + probe = (port) => probePort("0.0.0.0", port) + } = options; + + for (const service of services) { + const port = parsePort(service.port, service.label); + if (managedPorts.has(port)) continue; + const result = await probe(port); + if (!result.available) { + throw new Error(`${service.label} port ${port} is already in use`); + } + } +} diff --git a/scripts/deploy/ports.test.mjs b/scripts/deploy/ports.test.mjs new file mode 100644 index 00000000..49bd3a3f --- /dev/null +++ b/scripts/deploy/ports.test.mjs @@ -0,0 +1,136 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + parsePort, + selectDeploymentPort, + verifySelectedPorts +} from "./ports.mjs"; + +test("parsePort accepts valid integers", () => { + assert.equal(parsePort("3000", "Web"), 3000); + assert.equal(parsePort(8787, "API"), 8787); +}); + +test("parsePort rejects invalid values", () => { + assert.throws(() => parsePort("0", "Web"), /Web/); + assert.throws(() => parsePort("70000", "API"), /API/); + assert.throws(() => parsePort("abc", "Web"), /Web/); +}); + +test("loops until the requested alternative is available", async () => { + const answers = ["2", "3001", "3002"]; + const selected = await selectDeploymentPort({ + label: "Web", + defaultPort: 3000, + reserved: new Set([8787]), + managedPorts: new Set(), + ask: async () => answers.shift(), + probe: async (port) => ({ available: port === 3002, owner: port === 3001 ? "node pid=42" : null }) + }); + assert.equal(selected, 3002); +}); + +test("accepts available default without choosing alternative", async () => { + const selected = await selectDeploymentPort({ + label: "Web", + defaultPort: 3000, + reserved: new Set(), + managedPorts: new Set(), + ask: async () => "1", + probe: async () => ({ available: true, owner: null }), + print: () => {} + }); + assert.equal(selected, 3000); +}); + +test("rejects duplicate reserved ports", async () => { + await assert.rejects( + selectDeploymentPort({ + label: "Web", + defaultPort: 3000, + reserved: new Set([3000]), + managedPorts: new Set(), + nonInteractive: true, + ask: async () => assert.fail("must not prompt"), + probe: async () => ({ available: true, owner: null }) + }), + /already selected|reserved|冲突|duplicate/i + ); +}); + +test("non-interactive mode fails on an unknown listener", async () => { + await assert.rejects( + selectDeploymentPort({ + label: "API", + defaultPort: 8787, + reserved: new Set(), + managedPorts: new Set(), + nonInteractive: true, + ask: async () => assert.fail("must not prompt"), + probe: async () => ({ available: false, owner: "python pid=99" }) + }), + /API port 8787 is already in use/ + ); +}); + +test("quitting from occupied-port prompt exits safely", async () => { + await assert.rejects( + selectDeploymentPort({ + label: "Web", + defaultPort: 3000, + reserved: new Set(), + managedPorts: new Set(), + ask: async () => "q", + probe: async () => ({ available: false, owner: "node pid=42" }), + print: () => {} + }), + /cancelled|退出|abort/i + ); +}); + +test("managed ports may be selected during update preflight", async () => { + const selected = await selectDeploymentPort({ + label: "Web", + defaultPort: 3000, + reserved: new Set(), + managedPorts: new Set([3000]), + nonInteractive: true, + ask: async () => assert.fail("must not prompt"), + probe: async () => ({ available: false, owner: "datafoundry pid=1" }) + }); + assert.equal(selected, 3000); +}); + +test("verifySelectedPorts fails when a port becomes occupied", async () => { + await assert.rejects( + verifySelectedPorts( + [{ label: "Web", port: 3000 }], + { + managedPorts: new Set(), + probe: async () => ({ available: false, owner: "nginx pid=7" }) + } + ), + /Web port 3000 is already in use/ + ); +}); + +test("verifySelectedPorts skips ports still held by the managed stack", async () => { + let probed = false; + await verifySelectedPorts( + [ + { label: "Web", port: 3000 }, + { label: "API", port: 8787 } + ], + { + managedPorts: new Set([3000]), + probe: async (port) => { + if (port === 3000) { + probed = true; + return { available: false, owner: "datafoundry pid=1" }; + } + return { available: true, owner: null }; + } + } + ); + assert.equal(probed, false); +}); diff --git a/scripts/deploy/process-state.mjs b/scripts/deploy/process-state.mjs new file mode 100644 index 00000000..9bd96287 --- /dev/null +++ b/scripts/deploy/process-state.mjs @@ -0,0 +1,241 @@ +import { spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, open, readFile, rename, rm, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; + +const SENSITIVE_KEY = /KEY|SECRET|TOKEN|PASSWORD|COOKIE|AUTHORIZATION/i; + +export function deploymentPaths(root) { + return { + logsDir: path.join(root, "storage/logs"), + runDir: path.join(root, "storage/run"), + runtimeLog: path.join(root, "storage/logs/datafoundry.log"), + pidFile: path.join(root, "storage/run/datafoundry.pid"), + deploymentJson: path.join(root, "storage/run/deployment.json") + }; +} + +export function validateDeploymentState(state) { + if (!state || typeof state !== "object" || Array.isArray(state)) { + throw new Error("deployment state must be an object"); + } + for (const key of Object.keys(state)) { + if (SENSITIVE_KEY.test(key)) { + throw new Error(`sensitive field ${key}`); + } + } + return state; +} + +async function writeAtomicJson(filePath, value) { + await mkdir(path.dirname(filePath), { recursive: true }); + const tempPath = `${filePath}.${process.pid}.tmp`; + await writeFile(tempPath, `${JSON.stringify(value, null, 2)}\n`, { encoding: "utf8", mode: 0o600 }); + await rename(tempPath, filePath); +} + +export async function writeDeploymentState(root, state) { + validateDeploymentState(state); + const paths = deploymentPaths(root); + await mkdir(paths.runDir, { recursive: true }); + await writeAtomicJson(paths.deploymentJson, state); + await writeFile(paths.pidFile, `${state.pid}\n`, { encoding: "utf8", mode: 0o600 }); +} + +export async function readDeploymentState(root) { + const filePath = deploymentPaths(root).deploymentJson; + try { + const raw = await readFile(filePath, "utf8"); + return validateDeploymentState(JSON.parse(raw)); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } +} + +export function isProcessAlive(pid) { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +export async function rotateRuntimeLog(logPath, options = {}) { + const maxBytes = options.maxBytes ?? 20 * 1024 * 1024; + const retain = options.retain ?? 5; + await mkdir(path.dirname(logPath), { recursive: true }); + try { + const info = await stat(logPath); + if (info.size <= maxBytes) return; + } catch (error) { + if (error?.code === "ENOENT") { + await writeFile(logPath, "", { encoding: "utf8", mode: 0o600 }); + return; + } + throw error; + } + + await rm(`${logPath}.${retain}`, { force: true }); + for (let i = retain - 1; i >= 1; i -= 1) { + const from = `${logPath}.${i}`; + const to = `${logPath}.${i + 1}`; + if (existsSync(from)) await rename(from, to); + } + await rename(logPath, `${logPath}.1`); + await writeFile(logPath, "", { encoding: "utf8", mode: 0o600 }); +} + +async function readLaunchIdFromProcAsync(pid) { + try { + const environ = await readFile(`/proc/${pid}/environ`); + const match = /DATAFOUNDRY_LAUNCH_ID=([^\0]+)/.exec(environ.toString("utf8")); + return match?.[1] ?? null; + } catch { + return null; + } +} + +export async function verifyManagedProcessForStop(pid, expectedLaunchId, options = {}) { + if (process.platform !== "linux") { + return { allowed: true }; + } + + if (!expectedLaunchId) { + return { allowed: false, reason: "missing-expected-launch-id" }; + } + + const readLaunchId = options.readLaunchId ?? readLaunchIdFromProcAsync; + const launchId = await readLaunchId(pid); + + if (launchId === expectedLaunchId) { + return { allowed: true }; + } + if (launchId && launchId !== expectedLaunchId) { + return { allowed: false, reason: "launch-id-mismatch" }; + } + + // Never fall back to cmdline heuristics — an unverifiable launch marker must refuse stop. + return { allowed: false, reason: "launch-id-unverified" }; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export async function startManagedStack(root, options = {}) { + const existing = await readDeploymentState(root); + if (existing?.pid && isProcessAlive(existing.pid)) { + throw new Error(`DataFoundry is already running with pid ${existing.pid}`); + } + + const paths = deploymentPaths(root); + await mkdir(paths.logsDir, { recursive: true }); + await mkdir(paths.runDir, { recursive: true }); + await rotateRuntimeLog(paths.runtimeLog); + + const launchId = options.launchId ?? randomBytes(16).toString("hex"); + const command = options.command ?? "npm"; + const args = options.args ?? ["run", "start"]; + const env = { + ...(options.env ?? process.env), + DATAFOUNDRY_LAUNCH_ID: launchId + }; + + const logFd = await open(paths.runtimeLog, "a", 0o600); + let child; + try { + child = spawn(command, args, { + cwd: root, + env, + detached: true, + stdio: ["ignore", logFd.fd, logFd.fd] + }); + } finally { + await logFd.close(); + } + + child.unref(); + + const state = { + pid: child.pid, + pgid: child.pid, + launchId, + status: "starting", + startedAt: new Date().toISOString(), + commitSha: options.commitSha ?? null, + ports: options.ports ?? null + }; + await writeDeploymentState(root, state); + return state; +} + +export async function stopManagedStack(root, options = {}) { + const state = await readDeploymentState(root); + if (!state?.pid) return { stopped: false, reason: "not-running" }; + if (!isProcessAlive(state.pid)) { + await clearDeploymentState(root); + return { stopped: false, reason: "stale" }; + } + + if (process.platform === "linux") { + const verification = await verifyManagedProcessForStop(state.pid, state.launchId); + if (!verification.allowed) { + throw new Error( + `Refusing to signal pid ${state.pid}: launch marker does not identify DataFoundry` + ); + } + } + + try { + process.kill(-state.pid, "SIGTERM"); + } catch { + process.kill(state.pid, "SIGTERM"); + } + + const timeoutMs = options.timeoutMs ?? 15_000; + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (!isProcessAlive(state.pid)) { + await clearDeploymentState(root); + return { stopped: true, reason: "terminated" }; + } + await sleep(100); + } + + try { + process.kill(-state.pid, "SIGKILL"); + } catch { + try { + process.kill(state.pid, "SIGKILL"); + } catch { + // process may have exited between checks + } + } + + const killTimeoutMs = options.killTimeoutMs ?? 5_000; + const killDeadline = Date.now() + killTimeoutMs; + while (Date.now() < killDeadline) { + if (!isProcessAlive(state.pid)) { + await clearDeploymentState(root); + return { stopped: true, reason: "killed" }; + } + await sleep(100); + } + + if (!isProcessAlive(state.pid)) { + await clearDeploymentState(root); + return { stopped: true, reason: "killed" }; + } + + throw new Error(`Timed out waiting for pid ${state.pid} to exit after SIGKILL`); +} + +async function clearDeploymentState(root) { + const paths = deploymentPaths(root); + await rm(paths.deploymentJson, { force: true }); + await rm(paths.pidFile, { force: true }); +} diff --git a/scripts/deploy/process-state.test.mjs b/scripts/deploy/process-state.test.mjs new file mode 100644 index 00000000..b9385727 --- /dev/null +++ b/scripts/deploy/process-state.test.mjs @@ -0,0 +1,156 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, stat, writeFile, mkdir } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { + deploymentPaths, + isProcessAlive, + readDeploymentState, + rotateRuntimeLog, + startManagedStack, + stopManagedStack, + validateDeploymentState, + verifyManagedProcessForStop, + writeDeploymentState +} from "./process-state.mjs"; + +test("validateDeploymentState rejects sensitive fields", () => { + assert.throws( + () => validateDeploymentState({ pid: 123, API_KEY: "must-not-leak" }), + /sensitive field API_KEY/ + ); +}); + +test("writeDeploymentState persists non-sensitive state atomically", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-state-")); + const state = { + pid: 12345, + launchId: "launch-1", + status: "starting", + startedAt: "2026-07-22T00:00:00.000Z", + ports: { web: 3000, api: 8787 } + }; + await writeDeploymentState(root, state); + assert.deepEqual(await readDeploymentState(root), state); + const raw = await readFile(deploymentPaths(root).deploymentJson, "utf8"); + assert.doesNotMatch(raw, /SECRET|TOKEN|PASSWORD|KEY=/i); +}); + +test("isProcessAlive detects live and stale pids", () => { + assert.equal(isProcessAlive(process.pid), true); + assert.equal(isProcessAlive(99999999), false); +}); + +test("rotateRuntimeLog keeps five archives at 20 MiB", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-log-")); + const logPath = path.join(root, "datafoundry.log"); + await writeFile(logPath, "x".repeat(20 * 1024 * 1024 + 1)); + for (let i = 1; i <= 5; i += 1) { + await writeFile(`${logPath}.${i}`, `old-${i}`); + } + await rotateRuntimeLog(logPath, { maxBytes: 20 * 1024 * 1024, retain: 5 }); + await assert.rejects(() => readFile(`${logPath}.6`, "utf8"), /ENOENT/); + assert.match(await readFile(`${logPath}.1`, "utf8"), /x/); + assert.equal(await readFile(logPath, "utf8"), ""); +}); + +test("start and stop managed stack are idempotent and refuse foreign pids", async (t) => { + if (process.platform === "win32") { + t.skip("process-group semantics are Unix-only"); + return; + } + + const root = await mkdtemp(path.join(os.tmpdir(), "df-proc-")); + await mkdir(path.join(root, "storage"), { recursive: true }); + + const started = await startManagedStack(root, { + command: process.execPath, + args: ["-e", "setInterval(() => {}, 1000)"], + env: { ...process.env }, + ports: { web: 3000, api: 8787 }, + commitSha: "deadbeef" + }); + assert.equal(isProcessAlive(started.pid), true); + assert.equal((await readDeploymentState(root)).status, "starting"); + assert.equal((await readDeploymentState(root)).launchId, started.launchId); + const runtimeLogMode = (await stat(deploymentPaths(root).runtimeLog)).mode & 0o777; + assert.equal(runtimeLogMode, 0o600); + + await assert.rejects( + () => startManagedStack(root, { + command: process.execPath, + args: ["-e", "setInterval(() => {}, 1000)"] + }), + /already running/i + ); + + const foreignRoot = await mkdtemp(path.join(os.tmpdir(), "df-foreign-")); + await writeDeploymentState(foreignRoot, { + pid: process.pid, + launchId: "not-this-process", + status: "running", + startedAt: new Date().toISOString(), + ports: { web: 3000, api: 8787 } + }); + await assert.rejects(() => stopManagedStack(foreignRoot), /does not identify DataFoundry|launch marker/i); + + await stopManagedStack(root); + assert.equal(isProcessAlive(started.pid), false); + assert.equal(await readDeploymentState(root), null); + + await stopManagedStack(root); +}); + +test("verifyManagedProcessForStop refuses stop when launch id is unreadable", async (t) => { + if (process.platform !== "linux") { + t.skip("proc verification is Linux-only"); + return; + } + + const unverified = await verifyManagedProcessForStop(12345, "launch-1", { + readLaunchId: async () => null + }); + assert.equal(unverified.allowed, false); + assert.equal(unverified.reason, "launch-id-unverified"); + + const mismatch = await verifyManagedProcessForStop(process.pid, "expected-launch", { + readLaunchId: async () => "other-launch" + }); + assert.equal(mismatch.allowed, false); + assert.equal(mismatch.reason, "launch-id-mismatch"); + + const matched = await verifyManagedProcessForStop(12345, "launch-1", { + readLaunchId: async () => "launch-1" + }); + assert.equal(matched.allowed, true); + + const missingExpected = await verifyManagedProcessForStop(12345, "", { + readLaunchId: async () => "launch-1" + }); + assert.equal(missingExpected.allowed, false); + assert.equal(missingExpected.reason, "missing-expected-launch-id"); +}); + +test("stopManagedStack SIGKILLs and clears state when SIGTERM times out", async (t) => { + if (process.platform === "win32") { + t.skip("process-group semantics are Unix-only"); + return; + } + + const root = await mkdtemp(path.join(os.tmpdir(), "df-kill-")); + await mkdir(path.join(root, "storage"), { recursive: true }); + + const started = await startManagedStack(root, { + command: "bash", + args: ["-c", "trap '' TERM; while true; do sleep 1; done"], + env: { ...process.env }, + ports: { web: 3000, api: 8787 } + }); + assert.equal(isProcessAlive(started.pid), true); + + const result = await stopManagedStack(root, { timeoutMs: 200, killTimeoutMs: 5_000 }); + assert.equal(result.stopped, true); + assert.equal(isProcessAlive(started.pid), false); + assert.equal(await readDeploymentState(root), null); +}); diff --git a/scripts/deploy/smoke-native-deploy.mjs b/scripts/deploy/smoke-native-deploy.mjs new file mode 100644 index 00000000..5fc3365b --- /dev/null +++ b/scripts/deploy/smoke-native-deploy.mjs @@ -0,0 +1,223 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createServer } from "node:net"; +import { cp, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { createDeployLogWriter } from "./cli.mjs"; + +const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const FIXTURE_SECRET = "fixture-deploy-secret-at-least-32-chars"; + +async function reservePort() { + return new Promise((resolve, reject) => { + const server = createServer(); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address ? address.port : null; + server.close((error) => { + if (error) reject(error); + else resolve(port); + }); + }); + server.on("error", reject); + }); +} + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: options.cwd, + env: options.env ?? process.env, + encoding: "utf8", + timeout: options.timeout ?? 20 * 60_000 + }); + if (result.status !== 0) { + const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`; + throw new Error(`${command} ${args.join(" ")} failed (${result.status}):\n${output}`); + } + return result; +} + +async function httpGet(url) { + const response = await fetch(url, { redirect: "manual" }); + return { status: response.status, ok: response.status >= 200 && response.status < 400 }; +} + +function shouldCopy(source) { + const relative = path.relative(ROOT, source); + if (!relative || relative.startsWith("..")) return true; + const parts = relative.split(path.sep); + const blocked = new Set([ + ".git", + "node_modules", + "storage", + ".next", + "dist", + ".venv", + "venv", + ".worktrees", + "coverage" + ]); + if (parts.some((part) => blocked.has(part))) return false; + if (relative === ".env" || relative === path.join("apps", "web", ".env.local")) return false; + // Stale tsbuildinfo from the source tree can make tsc skip emit in the temp checkout (no dist/). + if (path.basename(source).endsWith(".tsbuildinfo")) return false; + return true; +} + +async function copyCheckout(destination) { + await cp(ROOT, destination, { + recursive: true, + filter: (source) => shouldCopy(source) + }); + await mkdir(path.join(destination, "storage"), { recursive: true }); + await writeFile(path.join(destination, "storage", "sentinel.txt"), "native-deploy-sentinel\n"); +} + +function scanForSecrets(text) { + if (text.includes(FIXTURE_SECRET)) { + throw new Error("Fixture secret leaked into logs or status output"); + } +} + +async function preserveFailureLogs(tempRoot) { + const sourceDir = path.join(tempRoot, "storage/logs"); + const targetDir = path.join(ROOT, "storage/logs"); + try { + const entries = await readdir(sourceDir); + await mkdir(targetDir, { recursive: true }); + for (const entry of entries) { + const content = await readFile(path.join(sourceDir, entry), "utf8"); + scanForSecrets(content); + await writeFile(path.join(targetDir, entry), content, { encoding: "utf8", mode: 0o600 }); + } + } catch (error) { + if (error?.code !== "ENOENT") { + console.error(`[smoke:native-deploy] failed to preserve logs: ${error.message}`); + } + } +} + +async function main() { + const webPort = await reservePort(); + const apiPort = await reservePort(); + const tempRoot = await mkdtemp(path.join(os.tmpdir(), "datafoundry-native-deploy-")); + let stopped = false; + + const env = { + ...process.env, + WEB_HOST: "127.0.0.1", + WEB_PORT: String(webPort), + API_HOST: "127.0.0.1", + API_PORT: String(apiPort), + AUTH_PUBLIC_BASE_URL: `http://127.0.0.1:${webPort}`, + AUTH_SESSION_SECRET: FIXTURE_SECRET, + SECRET_MASTER_KEY: FIXTURE_SECRET, + CI: "true", + SKIP_POSTINSTALL_BUILD: "1" + }; + + try { + console.log(`[smoke:native-deploy] checkout=${tempRoot}`); + console.log(`[smoke:native-deploy] web=${webPort} api=${apiPort}`); + await copyCheckout(tempRoot); + + // Inject fixture secret into the same redacting deploy-log path used in production. + const redactProbe = await createDeployLogWriter(tempRoot, { timestamp: "smoke-redact-probe" }); + await redactProbe.append( + [ + `AUTH_SESSION_SECRET=${FIXTURE_SECRET}`, + `Authorization: Bearer ${FIXTURE_SECRET}`, + `{"apiKey":"${FIXTURE_SECRET}"}`, + `https://user:${FIXTURE_SECRET}@example.com/callback`, + "npm ci output", + "" + ].join("\n") + ); + await redactProbe.finalize(); + const probeLog = await readFile( + path.join(tempRoot, "storage/logs/deploy-smoke-redact-probe.log"), + "utf8" + ); + scanForSecrets(probeLog); + assert.match(probeLog, /AUTH_SESSION_SECRET=\*+/); + assert.match(probeLog, /Authorization: Bearer \*+/i); + + run("bash", ["./deploy.sh", "deploy", "--non-interactive"], { + cwd: tempRoot, + env, + timeout: 25 * 60_000 + }); + + const status = run("bash", ["./deploy.sh", "status"], { cwd: tempRoot, env }); + scanForSecrets(`${status.stdout}\n${status.stderr}`); + assert.match(status.stdout, /进程\s+running/); + + const healthz = await httpGet(`http://127.0.0.1:${apiPort}/healthz`); + const ready = await httpGet(`http://127.0.0.1:${apiPort}/ready`); + const web = await httpGet(`http://127.0.0.1:${webPort}/`); + assert.equal(healthz.status, 200); + assert.equal(ready.status, 200); + assert.ok(web.ok, `Web should be reachable, got ${web.status}`); + + const envText = await readFile(path.join(tempRoot, ".env"), "utf8"); + assert.match(envText, new RegExp(`^AUTH_SESSION_SECRET=${FIXTURE_SECRET}$`, "m")); + assert.doesNotMatch(envText, /^LLM_API_KEY=.+$/m); + assert.doesNotMatch(envText, /^LLM_MODEL=.+$/m); + + run("bash", ["./deploy.sh", "restart"], { cwd: tempRoot, env, timeout: 10 * 60_000 }); + const statusAfterRestart = run("bash", ["./deploy.sh", "status"], { cwd: tempRoot, env }); + scanForSecrets(`${statusAfterRestart.stdout}\n${statusAfterRestart.stderr}`); + assert.match(statusAfterRestart.stdout, /进程\s+running/); + + const sentinel = await readFile(path.join(tempRoot, "storage", "sentinel.txt"), "utf8"); + assert.equal(sentinel, "native-deploy-sentinel\n"); + + run("bash", ["./deploy.sh", "stop"], { cwd: tempRoot, env }); + run("bash", ["./deploy.sh", "stop"], { cwd: tempRoot, env }); + stopped = true; + + const finalStatus = run("bash", ["./deploy.sh", "status"], { cwd: tempRoot, env }); + scanForSecrets(`${finalStatus.stdout}\n${finalStatus.stderr}`); + assert.match(finalStatus.stdout, /进程\s+stopped/); + assert.equal( + await readFile(path.join(tempRoot, "storage", "sentinel.txt"), "utf8"), + "native-deploy-sentinel\n" + ); + + for (const relative of [ + "storage/logs/datafoundry.log", + "storage/logs/deploy-latest.log", + "storage/logs/deploy-smoke-redact-probe.log" + ]) { + try { + const logs = await readFile(path.join(tempRoot, relative), "utf8"); + scanForSecrets(logs); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + } + + console.log("[smoke:native-deploy] ok"); + } catch (error) { + await preserveFailureLogs(tempRoot); + throw error; + } finally { + if (!stopped) { + try { + spawnSync("bash", ["./deploy.sh", "stop"], { + cwd: tempRoot, + env, + encoding: "utf8", + timeout: 60_000 + }); + } catch { + // best effort + } + } + await rm(tempRoot, { recursive: true, force: true }); + } +} + +await main(); diff --git a/scripts/stack-runner.mjs b/scripts/stack-runner.mjs new file mode 100644 index 00000000..06bf90cf --- /dev/null +++ b/scripts/stack-runner.mjs @@ -0,0 +1,122 @@ +import { execSync, spawn } from "node:child_process"; +import { existsSync } from "node:fs"; +import { loadEnvFile } from "node:process"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + formatStackEndpoints, + resolveStackRuntimeConfig, + webProcessEnvironment, +} from "./stack-runtime-config.mjs"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); + +export async function runStack({ mode, args = [] }) { + loadRootEnv(); + const apiOnly = args.includes("--api"); + const webOnly = args.includes("--web"); + const startApi = !webOnly || apiOnly; + const startWeb = !apiOnly || webOnly; + const runtimeConfig = resolveStackRuntimeConfig(); + + if (mode === "development") { + execSync("node scripts/ensure-dev-environment.mjs", { + cwd: root, + stdio: "inherit", + env: process.env, + shell: true, + }); + const ports = [ + ...(startApi ? [Number(runtimeConfig.API_PORT)] : []), + ...(startWeb ? [Number(runtimeConfig.WEB_PORT)] : []), + ]; + for (const port of ports) freePort(port); + } + + const children = []; + if (startApi) { + const command = + mode === "development" + ? ["--workspace", "@datafoundry/api", "run", "dev"] + : ["--prefix", "apps/api", "run", "start"]; + children.push(spawnProcess("DataFoundry API", "npm", command, { ...process.env, ...runtimeConfig })); + } + if (startWeb) { + const webScript = mode === "development" ? "dev" : "start"; + const command = + mode === "development" + ? ["--workspace", "@datafoundry/web", "run", webScript] + : ["--prefix", "apps/web", "run", webScript]; + const webEnv = { + ...process.env, + ...runtimeConfig, + ...webProcessEnvironment(runtimeConfig), + }; + children.push(spawnProcess("DataFoundry Web", "npm", command, webEnv)); + } + + if (children.length === 0) { + throw new Error("Nothing to start. Use --api and/or --web."); + } + + console.log(formatStackEndpoints(runtimeConfig, { startApi, startWeb })); + let shuttingDown = false; + const shutdown = (signal) => { + if (shuttingDown) return; + shuttingDown = true; + for (const { child } of children) { + if (!child.killed) child.kill(signal); + } + }; + + process.on("SIGINT", () => shutdown("SIGINT")); + process.on("SIGTERM", () => shutdown("SIGTERM")); + for (const { child, label } of children) { + child.on("exit", (code, signal) => { + if (shuttingDown || signal) return; + console.error(`[stack] ${label} exited with code ${code ?? "unknown"}.`); + shutdown("SIGTERM"); + process.exitCode = code && code !== 0 ? code : 1; + }); + } +} + +function loadRootEnv() { + const envPath = join(root, ".env"); + if (existsSync(envPath)) loadEnvFile(envPath); +} + +function spawnProcess(label, command, args, env) { + const child = spawn(command, args, { + cwd: root, + stdio: "inherit", + env, + shell: process.platform === "win32", + }); + child.on("error", (error) => console.error(`[stack] Unable to start ${label}: ${error.message}`)); + return { child, label }; +} + +function freePort(port) { + try { + if (process.platform === "win32") { + const output = execSync(`netstat -ano | findstr :${port}`, { + encoding: "utf8", + shell: true, + stdio: ["ignore", "pipe", "ignore"], + }); + const pids = new Set(); + for (const line of output.split(/\r?\n/u)) { + if (!/\bLISTENING\b/u.test(line)) continue; + const pid = line.trim().split(/\s+/u).at(-1); + if (pid && /^\d+$/u.test(pid) && pid !== "0") pids.add(pid); + } + for (const pid of pids) execSync(`taskkill /F /PID ${pid}`, { stdio: "ignore", shell: true }); + return; + } + execSync(`fuser -k ${port}/tcp 2>/dev/null || true`, { cwd: root, stdio: "ignore", shell: true }); + } catch { + // The port was already free. + } +} diff --git a/scripts/stack-runtime-config.mjs b/scripts/stack-runtime-config.mjs new file mode 100644 index 00000000..9ac88130 --- /dev/null +++ b/scripts/stack-runtime-config.mjs @@ -0,0 +1,27 @@ +function port(value, fallback, name) { + const parsed = Number(value ?? fallback); + if (!Number.isInteger(parsed) || parsed < 1 || parsed > 65535) { + throw new Error(`${name} must be an integer between 1 and 65535`); + } + return String(parsed); +} + +export function resolveStackRuntimeConfig(env = process.env) { + return { + API_HOST: env.API_HOST?.trim() || "127.0.0.1", + API_PORT: port(env.API_PORT, 8787, "API_PORT"), + WEB_HOST: env.WEB_HOST?.trim() || "0.0.0.0", + WEB_PORT: port(env.WEB_PORT, 3000, "WEB_PORT"), + }; +} + +export function webProcessEnvironment(config) { + return { HOSTNAME: config.WEB_HOST, PORT: config.WEB_PORT }; +} + +export function formatStackEndpoints(config, enabled) { + const lines = ["DataFoundry endpoints:"]; + if (enabled.startWeb) lines.push(` Web: http://127.0.0.1:${config.WEB_PORT}`); + if (enabled.startApi) lines.push(` API: http://${config.API_HOST}:${config.API_PORT}`); + return lines.join("\n"); +} diff --git a/scripts/stack-runtime-config.test.mjs b/scripts/stack-runtime-config.test.mjs new file mode 100644 index 00000000..c5b13694 --- /dev/null +++ b/scripts/stack-runtime-config.test.mjs @@ -0,0 +1,31 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + formatStackEndpoints, + resolveStackRuntimeConfig, + webProcessEnvironment, +} from "./stack-runtime-config.mjs"; + +test("uses native deployment defaults", () => { + const config = resolveStackRuntimeConfig({}); + assert.equal(config.API_HOST, "127.0.0.1"); + assert.equal(config.API_PORT, "8787"); + assert.equal(config.WEB_HOST, "0.0.0.0"); + assert.equal(config.WEB_PORT, "3000"); +}); + +test("projects configured host and port into the Next.js child", () => { + const config = resolveStackRuntimeConfig({ WEB_HOST: "127.0.0.1", WEB_PORT: "3310" }); + assert.deepEqual(webProcessEnvironment(config), { HOSTNAME: "127.0.0.1", PORT: "3310" }); +}); + +test("prints actual configured endpoints", () => { + const config = resolveStackRuntimeConfig({ API_PORT: "8877", WEB_PORT: "3310" }); + const output = formatStackEndpoints(config, { startApi: true, startWeb: true }); + assert.match(output, /http:\/\/127\.0\.0\.1:8877/); + assert.match(output, /http:\/\/127\.0\.0\.1:3310/); +}); + +test("rejects invalid ports", () => { + assert.throws(() => resolveStackRuntimeConfig({ WEB_PORT: "70000" }), /WEB_PORT/); +}); diff --git a/scripts/start.mjs b/scripts/start.mjs new file mode 100644 index 00000000..f05efd57 --- /dev/null +++ b/scripts/start.mjs @@ -0,0 +1,4 @@ +#!/usr/bin/env node +import { runStack } from "./stack-runner.mjs"; + +await runStack({ mode: "production", args: process.argv.slice(2) }); From 7319d28bcda305e64c76b29be9ebe76c9bbd6611 Mon Sep 17 00:00:00 2001 From: wjk <1471511095@qq.com> Date: Sun, 26 Jul 2026 16:03:31 +0800 Subject: [PATCH 2/2] =?UTF-8?q?fix(=E9=83=A8=E7=BD=B2):=20=E5=85=B3?= =?UTF-8?q?=E9=97=AD=E6=89=98=E7=AE=A1=E7=AB=AF=E5=8F=A3=E8=AF=AF=E5=88=A4?= =?UTF-8?q?=E3=80=81=E5=AF=86=E9=92=A5=E9=9D=99=E9=BB=98=E8=A6=86=E7=9B=96?= =?UTF-8?q?=E4=B8=8E=20launchId=20=E5=8F=8C=E5=8D=A1=E6=AD=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 仅在 verified running state 时标记 managed ports；redeploy 默认以磁盘密钥为准；launchId 校验失败时清除 stale 状态且不向错误进程发信号。附带端口精确匹配、postgres URL 脱敏与 .env 权限提示。 --- scripts/deploy/cli.mjs | 188 ++++++++++++++++++-------- scripts/deploy/cli.test.mjs | 186 +++++++++++++++++++++++-- scripts/deploy/config.mjs | 25 ++-- scripts/deploy/config.test.mjs | 28 +++- scripts/deploy/ports.mjs | 50 +++++-- scripts/deploy/ports.test.mjs | 40 +++++- scripts/deploy/process-state.mjs | 72 ++++++++-- scripts/deploy/process-state.test.mjs | 72 +++++++++- 8 files changed, 556 insertions(+), 105 deletions(-) diff --git a/scripts/deploy/cli.mjs b/scripts/deploy/cli.mjs index 041e8f62..e77e3c1d 100644 --- a/scripts/deploy/cli.mjs +++ b/scripts/deploy/cli.mjs @@ -1,6 +1,6 @@ import { spawn } from "node:child_process"; import { constants, existsSync } from "node:fs"; -import { access, appendFile, copyFile, mkdir, readFile, statfs, writeFile } from "node:fs/promises"; +import { access, appendFile, copyFile, mkdir, readFile, stat, statfs, writeFile } from "node:fs/promises"; import { createInterface } from "node:readline"; import os from "node:os"; import path from "node:path"; @@ -10,6 +10,7 @@ import { deploymentHelp, parseDeployArgs } from "./args.mjs"; import { ensureDeploymentEnvironment, isCompleteDeploymentConfig, + isPlaceholderSecret, parseDeploymentEnvironment, redactSensitiveText, renderWebEnvironment, @@ -22,6 +23,8 @@ import { collectDeploymentHealth, probeHttp, waitForDeployment } from "./health. import { probePort, selectDeploymentPort, verifySelectedPorts } from "./ports.mjs"; import { deploymentPaths, + healStaleDeploymentState, + inspectManagedRuntime, isProcessAlive, readDeploymentState, startManagedStack, @@ -30,6 +33,14 @@ import { writeDeploymentState } from "./process-state.mjs"; +const SECRET_OVERLAY_KEYS = ["AUTH_SESSION_SECRET", "SECRET_MASTER_KEY"]; +const PROCESS_OVERLAY_KEYS = [ + "WEB_HOST", + "WEB_PORT", + "API_HOST", + "API_PORT", + "AUTH_PUBLIC_BASE_URL" +]; const ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), "../.."); const TUI_ENTRY = "apps/tui/dist/index.js"; @@ -61,39 +72,69 @@ export async function inspectWritablePath(targetPath) { } } -function overlayProcessEnv(text, processEnv = {}) { +function envFlagEnabled(value) { + const normalized = String(value ?? "") + .trim() + .toLowerCase(); + return normalized === "1" || normalized === "true" || normalized === "yes"; +} + +/** + * Overlay process env onto disk .env text. + * Non-secret bind/port/public URL keys may always overlay. + * Secrets overlay only when disk value is empty/placeholder, or when explicitly allowed. + */ +export function overlayProcessEnv(text, processEnv = {}, options = {}) { const updates = {}; - for (const key of [ - "WEB_HOST", - "WEB_PORT", - "API_HOST", - "API_PORT", - "AUTH_PUBLIC_BASE_URL", - "AUTH_SESSION_SECRET", - "SECRET_MASTER_KEY" - ]) { + for (const key of PROCESS_OVERLAY_KEYS) { if (processEnv[key] != null && String(processEnv[key]).trim() !== "") { updates[key] = String(processEnv[key]); } } - return Object.keys(updates).length > 0 ? updateDeploymentEnvironment(text, updates) : text; + + const parsed = parseDeploymentEnvironment(text ?? ""); + const allowSecretOverlay = + options.allowProcessSecretOverlay === true || + envFlagEnabled(processEnv.DATAFOUNDRY_ALLOW_PROCESS_SECRET_OVERLAY); + + for (const key of SECRET_OVERLAY_KEYS) { + if (processEnv[key] == null || String(processEnv[key]).trim() === "") continue; + if (allowSecretOverlay || isPlaceholderSecret(parsed[key])) { + updates[key] = String(processEnv[key]); + } + } + + return Object.keys(updates).length > 0 ? updateDeploymentEnvironment(text ?? "", updates) : text; } -export function collectManagedPorts(env = {}, state = null) { +/** + * Ports are "managed" only when deployment.json exists, pid is alive, and + * (on Linux) launchId verification passes. Bare .env WEB_PORT/API_PORT are never owned. + */ +export async function collectManagedPorts(env = {}, state = null, options = {}) { const ports = new Set(); - for (const key of ["WEB_PORT", "API_PORT"]) { - const port = Number(env[key]); - if (Number.isInteger(port) && port > 0) ports.add(port); + void env; + + if (!state?.ports || typeof state.ports !== "object") return ports; + if (!state?.pid) return ports; + + const alive = options.isAlive ?? isProcessAlive; + if (!alive(state.pid)) return ports; + + if (process.platform === "linux" || options.forceLaunchIdCheck) { + const verification = await verifyManagedProcessForStop(state.pid, state.launchId, { + readLaunchId: options.readLaunchId, + forceLaunchIdCheck: options.forceLaunchIdCheck + }); + if (!verification.allowed) return ports; } - if (state?.ports && typeof state.ports === "object") { - for (const value of Object.values(state.ports)) { - const port = Number(value); - if (Number.isInteger(port) && port > 0) ports.add(port); - } + + for (const value of Object.values(state.ports)) { + const port = Number(value); + if (Number.isInteger(port) && port > 0) ports.add(port); } return ports; } - export function resolveAuthPublicBaseUrl(existing, oldWebPort, newWebPort) { const url = String(existing ?? "").trim(); if (!url) return `http://127.0.0.1:${newWebPort}`; @@ -220,7 +261,11 @@ export async function configureDeploymentInteractively(options) { } const deploymentState = root ? await readDeploymentState(root).catch(() => null) : null; - const managedPorts = collectManagedPorts(env, deploymentState); + const managedPorts = await collectManagedPorts( + env, + deploymentState, + options.collectManagedPortsOptions ?? {} + ); const reserved = new Set(); const webPort = await selectPortWithHint({ @@ -373,10 +418,17 @@ function createRealDeps(context) { }, async loadConfiguration() { const sourceText = await loadRootEnvText(root); - const overlaid = overlayProcessEnv(sourceText, process.env); + const overlaid = overlayProcessEnv(sourceText, process.env, { + allowProcessSecretOverlay: envFlagEnabled(process.env.DATAFOUNDRY_ALLOW_PROCESS_SECRET_OVERLAY) + }); const generateSecrets = parsed.command === "deploy"; const ensured = ensureDeploymentEnvironment(overlaid, { generateSecrets }); - config = { env: ensured.env, envText: ensured.text, sourceText, generatedKeys: ensured.generatedKeys }; + config = { + env: ensured.env, + envText: ensured.text, + sourceText, + generatedKeys: ensured.generatedKeys + }; return config; }, async preflight(current, options) { @@ -419,12 +471,18 @@ function createRealDeps(context) { }, async writeConfiguration(current) { await writeDeploymentConfiguration(root, current.envText, current.webText, { - backup: parsed.reconfigure + backup: parsed.reconfigure, + // Redeploy without --reconfigure must not silently lose disk secrets. + backupExistingSecrets: !parsed.reconfigure }); }, async isRunning() { - const state = await readDeploymentState(root); - return Boolean(state?.pid && isProcessAlive(state.pid)); + const inspection = await inspectManagedRuntime(root); + if (inspection.stale) { + await healStaleDeploymentState(root); + return false; + } + return inspection.running; }, async stop() { await stopManagedStack(root); @@ -474,11 +532,15 @@ function createRealDeps(context) { { label: "Web", port: current.env.WEB_PORT }, { label: "API", port: current.env.API_PORT } ]; - // After stop-old, managed ports may still be briefly held by our previous process. - // Treat them as owned so verify-ports-again does not false-fail during redeploy. + // Only verified running managed ports are owned. After stop-old, TIME_WAIT is + // handled by short retries inside verifySelectedPorts — not by treating .env as owned. const state = await readDeploymentState(root).catch(() => null); - const managedPorts = collectManagedPorts(current.env, state); - await verifySelectedPorts(services, { managedPorts }); + const managedPorts = await collectManagedPorts(current.env, state); + await verifySelectedPorts(services, { + managedPorts, + retries: 5, + retryDelayMs: 200 + }); }, async start(current) { await startManagedStack(root, { @@ -493,8 +555,8 @@ function createRealDeps(context) { try { return await waitForDeployment({ checkProcessAlive: async () => { - const state = await readDeploymentState(root); - return Boolean(state?.pid && isProcessAlive(state.pid)); + const inspection = await inspectManagedRuntime(root); + return inspection.running; }, apiBaseUrl: `http://127.0.0.1:${current.env.API_PORT}`, webUrl: `http://127.0.0.1:${current.env.WEB_PORT}` @@ -508,19 +570,11 @@ function createRealDeps(context) { } }, async markHealthy(current) { - const state = await readDeploymentState(root); - if (!state?.pid || !isProcessAlive(state.pid)) { + const inspection = await inspectManagedRuntime(root); + if (!inspection.running || !inspection.state?.pid) { throw new Error("Managed process is not alive; refusing to mark deployment healthy"); } - if (process.platform === "linux") { - const verification = await verifyManagedProcessForStop(state.pid, state.launchId); - if (!verification.allowed) { - throw new Error( - `Refusing to mark healthy: launch marker does not identify DataFoundry (${verification.reason})` - ); - } - } - await writeDeploymentState(root, { ...state, status: "healthy" }); + await writeDeploymentState(root, { ...inspection.state, status: "healthy" }); if (deployLog) await deployLog.finalize(); print(`DataFoundry is healthy at ${current.env.AUTH_PUBLIC_BASE_URL}`); print("Next: open Web, register/login, then create and enable a model profile."); @@ -530,12 +584,13 @@ function createRealDeps(context) { }, async status() { - const state = await readDeploymentState(root); + const inspection = await inspectManagedRuntime(root); + const state = inspection.state; const env = config?.env ?? ensureDeploymentEnvironment(await loadRootEnvText(root), { generateSecrets: false }).env; const summary = await collectDeploymentHealth({ - checkProcessAlive: async () => Boolean(state?.pid && isProcessAlive(state.pid)), + checkProcessAlive: async () => inspection.running, apiBaseUrl: `http://127.0.0.1:${env.API_PORT}`, webUrl: `http://127.0.0.1:${env.WEB_PORT}` }); @@ -594,8 +649,9 @@ export async function runDeploymentDoctor(root, options = {}) { record(`config: ${configIssues.length === 0 ? "ok" : configIssues.join(", ")}`); record(`config web=${env.WEB_PORT} api=${env.API_PORT} public=${env.AUTH_PUBLIC_BASE_URL}`); - const state = await readDeploymentState(root); - const managedPorts = collectManagedPorts(env, state); + const inspection = await inspectManagedRuntime(root); + const state = inspection.state; + const managedPorts = await collectManagedPorts(env, state); const portServices = [ { label: "web", port: env.WEB_PORT }, { label: "api", port: env.API_PORT } @@ -630,7 +686,23 @@ export async function runDeploymentDoctor(root, options = {}) { `permissions storage: not writable (${storageWritability.error ?? storageWritability.checkedPath})` ); } - record(`permissions .env: ${existsSync(envPath) ? "present" : "missing"}`); + if (existsSync(envPath)) { + try { + const mode = (await stat(envPath)).mode & 0o777; + const modeText = mode.toString(8).padStart(3, "0"); + if (mode === 0o600) { + record(`permissions .env: present mode=${modeText}`); + } else { + record( + `permissions .env: present mode=${modeText} (expected 600; run: chmod 600 .env)` + ); + } + } catch (error) { + record(`permissions .env: present (mode unavailable: ${error.message})`); + } + } else { + record("permissions .env: missing"); + } try { const diskPath = existsSync(storageRoot) ? storageRoot : root; @@ -643,12 +715,21 @@ export async function runDeploymentDoctor(root, options = {}) { if (state?.pid) { const alive = isProcessAlive(state.pid); - record(`pid: ${state.pid} status=${state.status ?? "unknown"} alive=${alive}`); + record( + `pid: ${state.pid} status=${state.status ?? "unknown"} alive=${alive} managed=${inspection.running}` + ); + if (inspection.stale) { + record( + `pid: stale (${inspection.reason}). Fix: ./deploy.sh stop # clears stale state without signaling foreign pid; then ./deploy.sh start` + ); + } + } else if (inspection.stale) { + record(`pid: stale state (${inspection.reason}). Fix: ./deploy.sh stop && ./deploy.sh start`); } else { record("pid: none"); } - if (state?.pid && isProcessAlive(state.pid)) { + if (inspection.running) { const summary = await collectDeploymentHealth( { processAlive: true, @@ -664,9 +745,8 @@ export async function runDeploymentDoctor(root, options = {}) { record("health: skipped (process not running)"); } - return { lines, env, state }; + return { lines, env, state, inspection }; } - export function formatDeploymentFailure(failure) { const summary = failure.summary ?? failure.error?.message ?? failure.stage; const lines = [`✗ ${summary}`]; diff --git a/scripts/deploy/cli.test.mjs b/scripts/deploy/cli.test.mjs index b48921ed..6e3117b9 100644 --- a/scripts/deploy/cli.test.mjs +++ b/scripts/deploy/cli.test.mjs @@ -179,15 +179,87 @@ test("reconfigure creates backup and keeps secrets", async () => { ); }); -test("collectManagedPorts gathers env and deployment state ports", () => { - const ports = collectManagedPorts( +test("collectManagedPorts ignores bare .env ports without a verified running state", async () => { + const bare = await collectManagedPorts( { WEB_PORT: "3310", API_PORT: "8877" }, - { ports: { web: 3310, api: 8877 } } + null + ); + assert.deepEqual([...bare], []); + + const deadPid = await collectManagedPorts( + { WEB_PORT: "3310", API_PORT: "8877" }, + { + pid: 99999999, + launchId: "launch-1", + ports: { web: 3310, api: 8877 } + } + ); + assert.deepEqual([...deadPid], []); +}); + +test("collectManagedPorts marks state ports only when pid is alive and launchId verifies", async () => { + const unmanaged = await collectManagedPorts( + { WEB_PORT: "3310", API_PORT: "8877" }, + { + pid: process.pid, + launchId: "expected-launch", + ports: { web: 3310, api: 8877 } + }, + { + readLaunchId: async () => "other-launch", + forceLaunchIdCheck: true + } + ); + assert.deepEqual([...unmanaged], []); + + const managed = await collectManagedPorts( + { WEB_PORT: "3000", API_PORT: "8787" }, + { + pid: process.pid, + launchId: "launch-1", + ports: { web: 3310, api: 8877 } + }, + { + readLaunchId: async () => "launch-1", + forceLaunchIdCheck: true + } + ); + assert.deepEqual([...managed].sort((a, b) => a - b), [3310, 8877]); +}); + +test("non-interactive fails when ports are occupied without a running managed state", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-occupied-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const source = [ + "WEB_PORT=3310", + "API_PORT=8877", + "AUTH_SESSION_SECRET=existing-session-secret-value", + "SECRET_MASTER_KEY=existing-master-secret-value", + "AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3310", + "DATAFOUNDRY_AUTH_MODE=password", + "AUTH_EMAIL_DELIVERY=test", + "WEB_HOST=0.0.0.0", + "API_HOST=127.0.0.1", + "STORAGE_ROOT_DIR=storage", + "METADATA_DB_PATH=storage/metadata/workbench.sqlite" + ].join("\n"); + + await assert.rejects( + () => + configureDeploymentInteractively({ + root, + sourceText: source, + reconfigure: false, + nonInteractive: true, + ask: async () => assert.fail("must not prompt"), + print: () => {}, + probe: async () => ({ available: false, owner: "nginx pid=42" }) + }), + /already in use/i ); - assert.deepEqual([...ports].sort((a, b) => a - b), [3310, 8877]); }); -test("reconfigure reuses managed listening ports during update preflight", async () => { +test("reconfigure reuses managed listening ports only for a verified running stack", async () => { const probes = []; const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-")); await mkdir(path.join(root, "apps/web"), { recursive: true }); @@ -208,8 +280,8 @@ test("reconfigure reuses managed listening ports during update preflight", async await writeFile( path.join(root, "storage/run/deployment.json"), `${JSON.stringify({ - pid: 99999, - launchId: "launch-1", + pid: process.pid, + launchId: "launch-managed", status: "healthy", startedAt: "2026-07-22T00:00:00.000Z", ports: { web: 3310, api: 8877 } @@ -224,7 +296,11 @@ test("reconfigure reuses managed listening ports during update preflight", async print: () => {}, probe: async (port) => { probes.push(port); - return { available: false, owner: "datafoundry pid=99999" }; + return { available: false, owner: `datafoundry pid=${process.pid}` }; + }, + collectManagedPortsOptions: { + readLaunchId: async () => "launch-managed", + forceLaunchIdCheck: true } }); assert.equal(result.env.WEB_PORT, "3310"); @@ -233,6 +309,55 @@ test("reconfigure reuses managed listening ports during update preflight", async assert.ok(probes.includes(8877)); }); +test("process env does not silently overwrite existing non-empty secrets", async () => { + const { overlayProcessEnv } = await import("./cli.mjs"); + const source = [ + "WEB_PORT=3000", + "API_PORT=8787", + "AUTH_SESSION_SECRET=disk-session-secret-value-32chars", + "SECRET_MASTER_KEY=disk-master-secret-value-32charsxx", + "AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3000" + ].join("\n"); + + const overlaid = overlayProcessEnv(source, { + WEB_PORT: "3310", + AUTH_SESSION_SECRET: "process-session-secret-should-not-win", + SECRET_MASTER_KEY: "process-master-secret-should-not-win" + }); + const env = (await import("./config.mjs")).parseDeploymentEnvironment(overlaid); + assert.equal(env.WEB_PORT, "3310"); + assert.equal(env.AUTH_SESSION_SECRET, "disk-session-secret-value-32chars"); + assert.equal(env.SECRET_MASTER_KEY, "disk-master-secret-value-32charsxx"); +}); + +test("process env may inject secrets only when disk is empty/placeholder or explicitly allowed", async () => { + const { overlayProcessEnv } = await import("./cli.mjs"); + const { parseDeploymentEnvironment } = await import("./config.mjs"); + + const fromPlaceholder = overlayProcessEnv( + "AUTH_SESSION_SECRET=change-me\nSECRET_MASTER_KEY=\n", + { + AUTH_SESSION_SECRET: "ci-session-secret-value-at-least-32", + SECRET_MASTER_KEY: "ci-master-secret-value-at-least-32x" + } + ); + const placeholderEnv = parseDeploymentEnvironment(fromPlaceholder); + assert.equal(placeholderEnv.AUTH_SESSION_SECRET, "ci-session-secret-value-at-least-32"); + assert.equal(placeholderEnv.SECRET_MASTER_KEY, "ci-master-secret-value-at-least-32x"); + + const explicit = overlayProcessEnv( + "AUTH_SESSION_SECRET=disk-session-secret-value-32chars\nSECRET_MASTER_KEY=disk-master-secret-value-32charsxx\n", + { + AUTH_SESSION_SECRET: "ci-session-secret-value-at-least-32", + SECRET_MASTER_KEY: "ci-master-secret-value-at-least-32x", + DATAFOUNDRY_ALLOW_PROCESS_SECRET_OVERLAY: "1" + }, + { allowProcessSecretOverlay: true } + ); + const explicitEnv = parseDeploymentEnvironment(explicit); + assert.equal(explicitEnv.AUTH_SESSION_SECRET, "ci-session-secret-value-at-least-32"); + assert.equal(explicitEnv.SECRET_MASTER_KEY, "ci-master-secret-value-at-least-32x"); +}); test("resolveAuthPublicBaseUrl preserves HTTPS proxy URLs without explicit port", () => { assert.equal( resolveAuthPublicBaseUrl("https://prod.example.com", "3000", "3001"), @@ -360,6 +485,51 @@ test("inspectWritablePath checks access without creating files", async () => { assert.equal(existsSync(path.join(root, "storage")), false); }); +test("runDeploymentDoctor reports actionable fix for stale launchId state", async (t) => { + if (process.platform !== "linux") { + t.skip("proc launchId verification is Linux-only"); + return; + } + + const root = await mkdtemp(path.join(os.tmpdir(), "df-cli-doctor-stale-")); + await mkdir(path.join(root, "storage/run"), { recursive: true }); + await writeFile( + path.join(root, ".env"), + [ + "WEB_PORT=3000", + "API_PORT=8787", + "AUTH_SESSION_SECRET=existing-session-secret-value", + "SECRET_MASTER_KEY=existing-master-secret-value", + "AUTH_PUBLIC_BASE_URL=http://127.0.0.1:3000" + ].join("\n") + ); + await writeFile( + path.join(root, "storage/run/deployment.json"), + `${JSON.stringify({ + pid: process.pid, + launchId: "not-this-process", + status: "healthy", + startedAt: "2026-07-26T00:00:00.000Z", + ports: { web: 3000, api: 8787 } + })}\n` + ); + + const lines = []; + await runDeploymentDoctor(root, { + print: (line) => lines.push(line), + run: async (command, args = []) => { + const key = [command, ...args].join(" "); + if (key === "node --version") return { stdout: "v22.14.0\n" }; + if (key === "npm --version") return { stdout: "10.9.0\n" }; + throw new Error(`missing mock for ${key}`); + }, + probe: async () => ({ available: true, owner: null }) + }); + const joined = lines.join("\n"); + assert.match(joined, /stale/i); + assert.match(joined, /\.\/deploy\.sh stop/); + assert.match(joined, /permissions \.env: present mode=/); +}); test("resolveTuiRuntimeUrl defaults to deployed API port", () => { assert.equal( resolveTuiRuntimeUrl({ API_PORT: "8877" }), diff --git a/scripts/deploy/config.mjs b/scripts/deploy/config.mjs index 983dc95c..e1c71943 100644 --- a/scripts/deploy/config.mjs +++ b/scripts/deploy/config.mjs @@ -164,17 +164,22 @@ export async function writeDeploymentConfiguration(root, rootText, webText, opti const envPath = path.join(root, ".env"); const webPath = path.join(root, "apps/web/.env.local"); let backupPath; + const { readFile } = await import("node:fs/promises"); - if (options.backup) { + let existing = ""; + try { + existing = await readFile(envPath, "utf8"); + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + + const existingEnv = parseDeploymentEnvironment(existing); + const hasExistingSecrets = SECRET_KEYS.some((key) => !isPlaceholderSecret(existingEnv[key])); + const shouldBackup = Boolean(options.backup) || Boolean(options.backupExistingSecrets && hasExistingSecrets); + + if (shouldBackup) { const stamp = options.timestamp ?? new Date().toISOString().replace(/[-:TZ.]/g, "").slice(0, 14); backupPath = path.join(root, `.env.backup-${stamp}`); - const { readFile } = await import("node:fs/promises"); - let existing = ""; - try { - existing = await readFile(envPath, "utf8"); - } catch (error) { - if (error?.code !== "ENOENT") throw error; - } await writeAtomic(backupPath, existing, 0o600); } @@ -182,7 +187,6 @@ export async function writeDeploymentConfiguration(root, rootText, webText, opti await writeAtomic(webPath, webText, 0o600); return { envPath, webPath, backupPath }; } - export function redactSensitiveText(text) { let result = String(text ?? ""); @@ -205,10 +209,9 @@ export function redactSensitiveText(text) { ); result = result.replace( - /(https?:\/\/)([^/\s:@]+):([^/\s@]+)@/gi, + /([a-z][a-z0-9+.-]*:\/\/)([^/\s:@]+):([^/\s@]+)@/gi, (_, protocol) => `${protocol}****:****@` ); - result = result.replace( /\b((?:sk|rk|pk|tok)-[A-Za-z0-9_-]{8,}|(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{16,}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g, (value) => maskSecret(value) diff --git a/scripts/deploy/config.test.mjs b/scripts/deploy/config.test.mjs index 7a7fcf01..ad469bfc 100644 --- a/scripts/deploy/config.test.mjs +++ b/scripts/deploy/config.test.mjs @@ -87,16 +87,42 @@ test("redactSensitiveText masks JSON keys, bearer tokens, URL userinfo, and toke '{"apiKey":"json-secret-value","api_key":"also-secret"}', "Authorization: Bearer fixture-deploy-secret-at-least-32-chars", "https://user:fixture-deploy-secret-at-least-32-chars@example.com/path", + "postgres://dbuser:dbpass@localhost:5432/app", "token sk-abcdefghijklmnop", "WEB_PORT=3000" ].join("\n") ); - assert.doesNotMatch(redacted, /json-secret-value|also-secret|fixture-deploy-secret-at-least-32-chars|sk-abcdefghijklmnop|user:fixture/); + assert.doesNotMatch(redacted, /json-secret-value|also-secret|fixture-deploy-secret-at-least-32-chars|sk-abcdefghijklmnop|user:fixture|dbuser:dbpass/); assert.match(redacted, /Authorization: Bearer \*+/i); assert.match(redacted, /https:\/\/\*\*\*\*:\*\*\*\*@example\.com\/path/); + assert.match(redacted, /postgres:\/\/\*\*\*\*:\*\*\*\*@localhost:5432\/app/); assert.match(redacted, /WEB_PORT=3000/); }); +test("writeDeploymentConfiguration backs up existing secrets even without reconfigure", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "datafoundry-config-secret-backup-")); + await mkdir(path.join(root, "apps/web"), { recursive: true }); + const existing = [ + "AUTH_SESSION_SECRET=existing-session-secret-value", + "SECRET_MASTER_KEY=existing-master-secret-value", + "WEB_PORT=3000" + ].join("\n"); + await writeFile(path.join(root, ".env"), `${existing}\n`); + + const next = [ + "AUTH_SESSION_SECRET=existing-session-secret-value", + "SECRET_MASTER_KEY=existing-master-secret-value", + "WEB_PORT=3310" + ].join("\n"); + const written = await writeDeploymentConfiguration(root, `${next}\n`, "API_PROXY_TARGET=http://127.0.0.1:8787\n", { + backup: false, + backupExistingSecrets: true, + timestamp: "20260726-120000" + }); + assert.ok(written.backupPath); + assert.match(await readFile(written.backupPath, "utf8"), /AUTH_SESSION_SECRET=existing-session-secret-value/); + assert.match(await readFile(path.join(root, ".env"), "utf8"), /WEB_PORT=3310/); +}); test("ensureDeploymentEnvironment can skip secret generation", () => { const result = ensureDeploymentEnvironment("WEB_PORT=3000\n", { generateSecrets: false }); assert.equal(result.env.AUTH_SESSION_SECRET, undefined); diff --git a/scripts/deploy/ports.mjs b/scripts/deploy/ports.mjs index f36b840b..dad38c2f 100644 --- a/scripts/deploy/ports.mjs +++ b/scripts/deploy/ports.mjs @@ -12,16 +12,27 @@ export function parsePort(value, label) { return parsed; } -export async function describePortOwner(port) { - if (process.platform !== "linux") return "unknown process"; +function lineOwnsPort(line, port) { + // Match ":3000" but not ":30001" — require a non-digit boundary after the port. + const pattern = new RegExp(`:${Number(port)}(?!\\d)`, "u"); + return pattern.test(line); +} + +export async function describePortOwner(port, options = {}) { + if (process.platform !== "linux" && !options.runSs) return "unknown process"; try { - const { stdout } = await execFileAsync("ss", ["-ltnp"], { - encoding: "utf8", - maxBuffer: 2 * 1024 * 1024 - }); - const needle = `:${port}`; - for (const line of stdout.split(/\r?\n/u)) { - if (!line.includes(needle) || !/\bLISTEN\b/u.test(line)) continue; + const runSs = + options.runSs ?? + (async () => { + const { stdout } = await execFileAsync("ss", ["-ltnp"], { + encoding: "utf8", + maxBuffer: 2 * 1024 * 1024 + }); + return stdout; + }); + const stdout = await runSs(); + for (const line of String(stdout ?? "").split(/\r?\n/u)) { + if (!/\bLISTEN\b/u.test(line) || !lineOwnsPort(line, port)) continue; const users = /users:\(\("([^"]+)",pid=(\d+)/u.exec(line); if (users) return `${users[1]} pid=${users[2]}`; return "unknown process"; @@ -31,7 +42,6 @@ export async function describePortOwner(port) { } return "unknown process"; } - export async function probePort(host, port, options = {}) { const describe = options.describeOwner ?? describePortOwner; const available = await new Promise((resolve) => { @@ -110,18 +120,30 @@ export async function selectDeploymentPort(options) { } } +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + export async function verifySelectedPorts(services, options = {}) { const { managedPorts = new Set(), - probe = (port) => probePort("0.0.0.0", port) + probe = (port) => probePort("0.0.0.0", port), + retries = 5, + retryDelayMs = 200 } = options; for (const service of services) { const port = parsePort(service.port, service.label); if (managedPorts.has(port)) continue; - const result = await probe(port); - if (!result.available) { + + let lastResult = null; + for (let attempt = 0; attempt <= retries; attempt += 1) { + lastResult = await probe(port); + if (lastResult.available) break; + if (attempt < retries) await sleep(retryDelayMs); + } + if (!lastResult?.available) { throw new Error(`${service.label} port ${port} is already in use`); } } -} +} \ No newline at end of file diff --git a/scripts/deploy/ports.test.mjs b/scripts/deploy/ports.test.mjs index 49bd3a3f..f42e4cac 100644 --- a/scripts/deploy/ports.test.mjs +++ b/scripts/deploy/ports.test.mjs @@ -1,11 +1,11 @@ import assert from "node:assert/strict"; import test from "node:test"; import { + describePortOwner, parsePort, selectDeploymentPort, verifySelectedPorts } from "./ports.mjs"; - test("parsePort accepts valid integers", () => { assert.equal(parsePort("3000", "Web"), 3000); assert.equal(parsePort(8787, "API"), 8787); @@ -107,13 +107,13 @@ test("verifySelectedPorts fails when a port becomes occupied", async () => { [{ label: "Web", port: 3000 }], { managedPorts: new Set(), + retries: 0, probe: async () => ({ available: false, owner: "nginx pid=7" }) } ), /Web port 3000 is already in use/ ); }); - test("verifySelectedPorts skips ports still held by the managed stack", async () => { let probed = false; await verifySelectedPorts( @@ -134,3 +134,39 @@ test("verifySelectedPorts skips ports still held by the managed stack", async () ); assert.equal(probed, false); }); + +test("verifySelectedPorts retries briefly for transient TIME_WAIT occupancy", async () => { + let attempts = 0; + await verifySelectedPorts( + [{ label: "Web", port: 3000 }], + { + managedPorts: new Set(), + retries: 3, + retryDelayMs: 1, + probe: async () => { + attempts += 1; + if (attempts < 3) return { available: false, owner: "unknown process" }; + return { available: true, owner: null }; + } + } + ); + assert.equal(attempts, 3); +}); + +test("describePortOwner matches exact port and not :30001 for :3000", async () => { + const owner = await describePortOwner(3000, { + runSs: async () => + [ + "State Recv-Q Send-Q Local Address:Port Peer Address:Port Process", + 'LISTEN 0 511 0.0.0.0:30001 0.0.0.0:* users:(("other",pid=9,fd=3))', + 'LISTEN 0 511 0.0.0.0:3000 0.0.0.0:* users:(("node",pid=42,fd=3))' + ].join("\n") + }); + assert.equal(owner, "node pid=42"); + + const noMatch = await describePortOwner(3000, { + runSs: async () => + 'LISTEN 0 511 0.0.0.0:30001 0.0.0.0:* users:(("other",pid=9,fd=3))\n' + }); + assert.equal(noMatch, "unknown process"); +}); \ No newline at end of file diff --git a/scripts/deploy/process-state.mjs b/scripts/deploy/process-state.mjs index 9bd96287..cb2e8b61 100644 --- a/scripts/deploy/process-state.mjs +++ b/scripts/deploy/process-state.mjs @@ -100,7 +100,7 @@ async function readLaunchIdFromProcAsync(pid) { } export async function verifyManagedProcessForStop(pid, expectedLaunchId, options = {}) { - if (process.platform !== "linux") { + if (process.platform !== "linux" && !options.forceLaunchIdCheck) { return { allowed: true }; } @@ -126,10 +126,56 @@ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } +async function clearDeploymentState(root) { + const paths = deploymentPaths(root); + await rm(paths.deploymentJson, { force: true }); + await rm(paths.pidFile, { force: true }); +} + +/** + * Returns whether the recorded deployment process is a verified managed stack. + * Alive pid + failed launchId verification is treated as stale (not running). + */ +export async function inspectManagedRuntime(root, options = {}) { + const state = await readDeploymentState(root); + if (!state?.pid) { + return { state: null, running: false, stale: false, reason: "not-running" }; + } + if (!isProcessAlive(state.pid)) { + return { state, running: false, stale: true, reason: "dead-pid" }; + } + + if (process.platform === "linux" || options.forceLaunchIdCheck) { + const verification = await verifyManagedProcessForStop(state.pid, state.launchId, options); + if (!verification.allowed) { + return { + state, + running: false, + stale: true, + reason: verification.reason ?? "stale-launch-id" + }; + } + } + + return { state, running: true, stale: false, reason: "running" }; +} + +export async function healStaleDeploymentState(root, options = {}) { + const inspection = await inspectManagedRuntime(root, options); + if (!inspection.stale) return inspection; + await clearDeploymentState(root); + return { ...inspection, state: null, healed: true }; +} + export async function startManagedStack(root, options = {}) { const existing = await readDeploymentState(root); if (existing?.pid && isProcessAlive(existing.pid)) { - throw new Error(`DataFoundry is already running with pid ${existing.pid}`); + const healed = await healStaleDeploymentState(root, options); + if (!healed.stale) { + throw new Error(`DataFoundry is already running with pid ${existing.pid}`); + } + } else if (existing?.pid && !isProcessAlive(existing.pid)) { + await clearDeploymentState(root); } const paths = deploymentPaths(root); @@ -181,12 +227,16 @@ export async function stopManagedStack(root, options = {}) { return { stopped: false, reason: "stale" }; } - if (process.platform === "linux") { - const verification = await verifyManagedProcessForStop(state.pid, state.launchId); + if (process.platform === "linux" || options.forceLaunchIdCheck) { + const verification = await verifyManagedProcessForStop(state.pid, state.launchId, options); if (!verification.allowed) { - throw new Error( - `Refusing to signal pid ${state.pid}: launch marker does not identify DataFoundry` - ); + // PID reuse / foreign process: never signal, but clear stale state so start can recover. + await clearDeploymentState(root); + return { + stopped: false, + reason: "stale-launch-id", + detail: verification.reason + }; } } @@ -232,10 +282,4 @@ export async function stopManagedStack(root, options = {}) { } throw new Error(`Timed out waiting for pid ${state.pid} to exit after SIGKILL`); -} - -async function clearDeploymentState(root) { - const paths = deploymentPaths(root); - await rm(paths.deploymentJson, { force: true }); - await rm(paths.pidFile, { force: true }); -} +} \ No newline at end of file diff --git a/scripts/deploy/process-state.test.mjs b/scripts/deploy/process-state.test.mjs index b9385727..b1ce7cac 100644 --- a/scripts/deploy/process-state.test.mjs +++ b/scripts/deploy/process-state.test.mjs @@ -93,7 +93,11 @@ test("start and stop managed stack are idempotent and refuse foreign pids", asyn startedAt: new Date().toISOString(), ports: { web: 3000, api: 8787 } }); - await assert.rejects(() => stopManagedStack(foreignRoot), /does not identify DataFoundry|launch marker/i); + const staleStop = await stopManagedStack(foreignRoot); + assert.equal(staleStop.stopped, false); + assert.match(String(staleStop.reason), /stale/i); + assert.equal(await readDeploymentState(foreignRoot), null); + assert.equal(isProcessAlive(process.pid), true); await stopManagedStack(root); assert.equal(isProcessAlive(started.pid), false); @@ -102,6 +106,72 @@ test("start and stop managed stack are idempotent and refuse foreign pids", asyn await stopManagedStack(root); }); +test("launchId mismatch is treated as stale: clear state, never signal, allow restart", async (t) => { + if (process.platform !== "linux") { + t.skip("proc launchId verification is Linux-only"); + return; + } + + const root = await mkdtemp(path.join(os.tmpdir(), "df-stale-launch-")); + await mkdir(path.join(root, "storage"), { recursive: true }); + + // Reuse this test process as a "foreign" live pid with mismatched launchId. + await writeDeploymentState(root, { + pid: process.pid, + launchId: "stale-or-reused-launch-id", + status: "healthy", + startedAt: new Date().toISOString(), + ports: { web: 3000, api: 8787 } + }); + + const healed = await stopManagedStack(root); + assert.equal(healed.stopped, false); + assert.match(String(healed.reason), /stale/i); + assert.equal(await readDeploymentState(root), null); + assert.equal(isProcessAlive(process.pid), true); + + const started = await startManagedStack(root, { + command: process.execPath, + args: ["-e", "setInterval(() => {}, 1000)"], + env: { ...process.env }, + ports: { web: 3000, api: 8787 } + }); + assert.equal(isProcessAlive(started.pid), true); + assert.notEqual(started.pid, process.pid); + + await stopManagedStack(root); + assert.equal(isProcessAlive(started.pid), false); + assert.equal(await readDeploymentState(root), null); +}); +test("startManagedStack self-heals stale launchId without signaling the foreign pid", async (t) => { + if (process.platform !== "linux") { + t.skip("proc launchId verification is Linux-only"); + return; + } + + const root = await mkdtemp(path.join(os.tmpdir(), "df-start-heal-")); + await mkdir(path.join(root, "storage"), { recursive: true }); + await writeDeploymentState(root, { + pid: process.pid, + launchId: "not-our-launch", + status: "healthy", + startedAt: new Date().toISOString(), + ports: { web: 3000, api: 8787 } + }); + + const started = await startManagedStack(root, { + command: process.execPath, + args: ["-e", "setInterval(() => {}, 1000)"], + env: { ...process.env }, + ports: { web: 3310, api: 8877 } + }); + assert.equal(isProcessAlive(process.pid), true); + assert.equal(isProcessAlive(started.pid), true); + assert.notEqual(started.pid, process.pid); + assert.equal((await readDeploymentState(root)).pid, started.pid); + + await stopManagedStack(root); +}); test("verifyManagedProcessForStop refuses stop when launch id is unreadable", async (t) => { if (process.platform !== "linux") { t.skip("proc verification is Linux-only");