From d79c3bcf1b30d529e50fc25d1b463b0fe9be95e3 Mon Sep 17 00:00:00 2001 From: im10furry Date: Mon, 6 Jul 2026 03:45:30 -0500 Subject: [PATCH] feat: add high-value backend modules --- .github/workflows/ci.yml | 9 + .github/workflows/release.yml | 3 + AGENTS.md | 17 +- AGENTS_zh.md | 17 +- README.md | 10 +- README_zh.md | 10 +- docs/.vitepress/config.ts | 10 +- docs/contributing.md | 4 +- docs/index.md | 12 + docs/modules/index.md | 7 +- docs/modules/metrics.md | 40 +++ docs/modules/migrations.md | 40 +++ docs/modules/problem.md | 40 +++ docs/zh/contributing.md | 2 +- docs/zh/index.md | 12 + docs/zh/modules/index.md | 7 +- docs/zh/modules/metrics.md | 40 +++ docs/zh/modules/migrations.md | 40 +++ docs/zh/modules/problem.md | 40 +++ internal/bundle/manifest.json | 268 +++++++++++++++- internal/bundle/registry.zip | Bin 720833 -> 802676 bytes registry/index.json | 53 +++- registry/metrics/README.md | 54 ++++ registry/metrics/__llms__.md | 3 + registry/metrics/src/go/config.go | 112 +++++++ registry/metrics/src/go/config_test.go | 29 ++ registry/metrics/src/go/go.mod | 17 + registry/metrics/src/go/go.sum | 24 ++ registry/metrics/src/go/handler.go | 16 + registry/metrics/src/go/handler_test.go | 33 ++ registry/metrics/src/go/metrics.go | 157 ++++++++++ registry/metrics/src/go/metrics_test.go | 63 ++++ registry/metrics/src/go/middleware.go | 66 ++++ registry/metrics/src/go/middleware_test.go | 78 +++++ registry/metrics/src/go/pentest_test.go | 56 ++++ registry/migrations/README.md | 67 ++++ registry/migrations/__llms__.md | 3 + registry/migrations/src/go/config.go | 135 ++++++++ registry/migrations/src/go/config_test.go | 62 ++++ registry/migrations/src/go/file.go | 253 +++++++++++++++ registry/migrations/src/go/file_test.go | 55 ++++ registry/migrations/src/go/go.mod | 3 + registry/migrations/src/go/handler.go | 31 ++ registry/migrations/src/go/handler_test.go | 45 +++ registry/migrations/src/go/migrator.go | 324 ++++++++++++++++++++ registry/migrations/src/go/migrator_test.go | 288 +++++++++++++++++ registry/migrations/src/go/pentest_test.go | 48 +++ registry/problem/README.md | 56 ++++ registry/problem/__llms__.md | 3 + registry/problem/src/go/config.go | 93 ++++++ registry/problem/src/go/config_test.go | 26 ++ registry/problem/src/go/go.mod | 3 + registry/problem/src/go/handler.go | 124 ++++++++ registry/problem/src/go/handler_test.go | 92 ++++++ registry/problem/src/go/pentest_test.go | 47 +++ registry/problem/src/go/problem.go | 182 +++++++++++ registry/problem/src/go/problem_test.go | 45 +++ 57 files changed, 3342 insertions(+), 32 deletions(-) create mode 100644 docs/modules/metrics.md create mode 100644 docs/modules/migrations.md create mode 100644 docs/modules/problem.md create mode 100644 docs/zh/modules/metrics.md create mode 100644 docs/zh/modules/migrations.md create mode 100644 docs/zh/modules/problem.md create mode 100644 registry/metrics/README.md create mode 100644 registry/metrics/__llms__.md create mode 100644 registry/metrics/src/go/config.go create mode 100644 registry/metrics/src/go/config_test.go create mode 100644 registry/metrics/src/go/go.mod create mode 100644 registry/metrics/src/go/go.sum create mode 100644 registry/metrics/src/go/handler.go create mode 100644 registry/metrics/src/go/handler_test.go create mode 100644 registry/metrics/src/go/metrics.go create mode 100644 registry/metrics/src/go/metrics_test.go create mode 100644 registry/metrics/src/go/middleware.go create mode 100644 registry/metrics/src/go/middleware_test.go create mode 100644 registry/metrics/src/go/pentest_test.go create mode 100644 registry/migrations/README.md create mode 100644 registry/migrations/__llms__.md create mode 100644 registry/migrations/src/go/config.go create mode 100644 registry/migrations/src/go/config_test.go create mode 100644 registry/migrations/src/go/file.go create mode 100644 registry/migrations/src/go/file_test.go create mode 100644 registry/migrations/src/go/go.mod create mode 100644 registry/migrations/src/go/handler.go create mode 100644 registry/migrations/src/go/handler_test.go create mode 100644 registry/migrations/src/go/migrator.go create mode 100644 registry/migrations/src/go/migrator_test.go create mode 100644 registry/migrations/src/go/pentest_test.go create mode 100644 registry/problem/README.md create mode 100644 registry/problem/__llms__.md create mode 100644 registry/problem/src/go/config.go create mode 100644 registry/problem/src/go/config_test.go create mode 100644 registry/problem/src/go/go.mod create mode 100644 registry/problem/src/go/handler.go create mode 100644 registry/problem/src/go/handler_test.go create mode 100644 registry/problem/src/go/pentest_test.go create mode 100644 registry/problem/src/go/problem.go create mode 100644 registry/problem/src/go/problem_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2889937..47ea9f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,9 @@ jobs: cache pagination mail + migrations + metrics + problem ) for mod in "${modules[@]}"; do echo "::group::go vet: $mod" @@ -110,6 +113,9 @@ jobs: cache pagination mail + migrations + metrics + problem ) failed=0 for mod in "${modules[@]}"; do @@ -158,6 +164,9 @@ jobs: cache pagination mail + migrations + metrics + problem ) for mod in "${modules[@]}"; do echo "::group::go build: $mod" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9a15542..96db1e0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -162,6 +162,9 @@ jobs: cache pagination mail + migrations + metrics + problem ) for mod in "${modules[@]}"; do diff --git a/AGENTS.md b/AGENTS.md index 3b50ad8..c7f1dd1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,12 +6,12 @@ ## Project Description -Scion is a copy-paste code library for Go backend development. It contains 12 self-contained modules in `registry/` — each is a standalone Go package. Modules are standard-library only by default; security-sensitive modules may be declared as `stdlibOnly:false` in `registry/index.json` and copied in standalone mode. Modules are meant to be copied into a user's project and adapted, not imported as a dependency. +Scion is a copy-paste code library for Go backend development. It contains 15 self-contained modules in `registry/` — each is a standalone Go package. Modules are standard-library only by default; security-sensitive or observability modules may be declared as `stdlibOnly:false` in `registry/index.json` and copied in standalone mode. Modules are meant to be copied into a user's project and adapted, not imported as a dependency. ## Coding Standards - Go 1.22+ with generics -- Standard library only by default; declared `stdlibOnly:false` modules may use mature security libraries +- Standard library only by default; declared `stdlibOnly:false` modules may use mature security or observability libraries - `gofmt` formatting is mandatory - `go vet` must pass with zero warnings - Middleware signature: `func(http.Handler) http.Handler` @@ -63,13 +63,13 @@ cd registry//src/go && go test -v -count=1 ./... ```bash # PowerShell -$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') +$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail','migrations','metrics','problem') foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Pop-Location } ``` ## Key Constraints -- Do NOT add external dependencies to any module's `go.mod` unless the module is explicitly marked `stdlibOnly:false` in `registry/index.json` and the dependency is justified for security or correctness +- Do NOT add external dependencies to any module's `go.mod` unless the module is explicitly marked `stdlibOnly:false` in `registry/index.json` and the dependency is justified for security, correctness, or observability - Do NOT use `panic` in HTTP handlers — return errors - Do NOT trust client headers (`Content-Type`, `X-Forwarded-For`, `X-Real-Ip`) - Do NOT use `strings.Split` for header parsing — use `strings.SplitN` with a limit @@ -89,7 +89,7 @@ You are working on Scion, a copy-paste code library for Go backend development. Project location: Architecture: -- 12 modules in registry/ — each is a standalone Go package +- 15 modules in registry/ — each is a standalone Go package - Module path pattern: registry//src/go/ - Go 1.22+, standard library by default, gofmt mandatory @@ -125,7 +125,7 @@ Task: ``` scion/ -├── registry/ # 12 copy-paste modules +├── registry/ # 15 copy-paste modules │ ├── auth/ # JWT auth + bcrypt │ ├── crud/ # Generic CRUD + pagination │ ├── database/ # database/sql helpers @@ -137,7 +137,10 @@ scion/ │ ├── health/ # Liveness/readiness probes │ ├── cache/ # TTL + LRU cache │ ├── pagination/ # Offset + cursor pagination -│ └── mail/ # SMTP email sender +│ ├── mail/ # SMTP email sender +│ ├── migrations/ # SQL migration runner +│ ├── metrics/ # Prometheus HTTP metrics +│ └── problem/ # RFC 9457 problem responses ├── docs/ # Human-readable docs ├── AGENTS.md # This file (English) ├── AGENTS_zh.md # This file (Chinese) diff --git a/AGENTS_zh.md b/AGENTS_zh.md index fe30e47..ae99ffa 100644 --- a/AGENTS_zh.md +++ b/AGENTS_zh.md @@ -6,12 +6,12 @@ ## 项目描述 -Scion 是一个面向 Go 后端开发的复制粘贴代码库。`registry/` 目录下有 12 个自包含模块，每个都是独立的 Go 包。模块默认仅使用标准库；安全敏感模块可以在 `registry/index.json` 中显式标记为 `stdlibOnly:false`，并以 standalone 模式复制。模块的设计目的是被复制到用户项目中并适配，而非作为依赖导入。 +Scion 是一个面向 Go 后端开发的复制粘贴代码库。`registry/` 目录下有 15 个自包含模块，每个都是独立的 Go 包。模块默认仅使用标准库；安全敏感或可观测性模块可以在 `registry/index.json` 中显式标记为 `stdlibOnly:false`，并以 standalone 模式复制。模块的设计目的是被复制到用户项目中并适配，而非作为依赖导入。 ## 编码标准 - Go 1.22+，使用泛型 -- 默认仅使用标准库；显式标记为 `stdlibOnly:false` 的模块可以使用成熟安全库 +- 默认仅使用标准库；显式标记为 `stdlibOnly:false` 的模块可以使用成熟安全或可观测性库 - `gofmt` 格式化是强制的 - `go vet` 必须零警告通过 - 中间件签名：`func(http.Handler) http.Handler` @@ -63,13 +63,13 @@ cd registry//src/go && go test -v -count=1 ./... ```bash # PowerShell -$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') +$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail','migrations','metrics','problem') foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Pop-Location } ``` ## 关键约束 -- 不要给任何模块的 `go.mod` 添加外部依赖，除非该模块已在 `registry/index.json` 显式标记为 `stdlibOnly:false`，且依赖对安全或正确性有明确价值 +- 不要给任何模块的 `go.mod` 添加外部依赖，除非该模块已在 `registry/index.json` 显式标记为 `stdlibOnly:false`，且依赖对安全、正确性或可观测性有明确价值 - 不要在 HTTP 处理器中使用 `panic` — 返回错误 - 不要信任客户端 header（`Content-Type`、`X-Forwarded-For`、`X-Real-Ip`） - 不要用 `strings.Split` 解析 header — 用 `strings.SplitN` 加限制 @@ -89,7 +89,7 @@ foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Po 项目路径: 架构: -- registry/ 下有 12 个模块 — 每个是独立的 Go 包 +- registry/ 下有 15 个模块 — 每个是独立的 Go 包 - 模块路径模式: registry/<模块名>/src/go/ - Go 1.22+，默认仅使用标准库，gofmt 强制 @@ -125,7 +125,7 @@ foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Po ``` scion/ -├── registry/ # 12 个复制粘贴模块 +├── registry/ # 15 个复制粘贴模块 │ ├── auth/ # JWT 认证 + bcrypt │ ├── crud/ # 泛型 CRUD + 分页 │ ├── database/ # database/sql 工具 @@ -137,7 +137,10 @@ scion/ │ ├── health/ # 存活/就绪探针 │ ├── cache/ # TTL + LRU 缓存 │ ├── pagination/ # Offset + cursor 分页 -│ └── mail/ # SMTP 邮件发送 +│ ├── mail/ # SMTP 邮件发送 +│ ├── migrations/ # SQL 迁移执行器 +│ ├── metrics/ # Prometheus HTTP 指标 +│ └── problem/ # RFC 9457 错误响应 ├── docs/ # 人类可读文档 ├── AGENTS.md # 本文件（英文） ├── AGENTS_zh.md # 本文件（中文） diff --git a/README.md b/README.md index b5bd143..9122df7 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ scion diff cache When `--to` or `--target` is omitted, Scion uses the module's default target, such as `internal/cache`. You can still override it with `--to ` or `--target `. -Scion copies source files and writes `.scion-module.json` metadata for later comparison. It never edits your project's `go.mod` automatically. Modules marked `stdlibOnly=false`, such as `auth`, require explicit standalone mode: +Scion copies source files and writes `.scion-module.json` metadata for later comparison. It never edits your project's `go.mod` automatically. Modules marked `stdlibOnly=false`, such as `auth` and `metrics`, require explicit standalone mode: ```bash scion add auth --standalone @@ -84,6 +84,9 @@ Backend modules such as auth, CRUD, file upload, and rate limiting share most of | [cache](registry/cache/) | Generic TTL + LRU cache | Background cleanup, goroutine leak prevention, max entries limit | | [pagination](registry/pagination/) | Offset/limit + cursor pagination | Cursor base64 validation, negative offset clamp, max limit enforcement | | [mail](registry/mail/) | SMTP email with templates | Header injection prevention, XSS escaping, attachment sanitization, async queue | +| [migrations](registry/migrations/) | SQL migration runner | Safe filenames, table whitelist, checksums, transactions | +| [metrics](registry/metrics/) | Prometheus HTTP metrics | Route cardinality limits, label sanitization, isolated registry | +| [problem](registry/problem/) | RFC 9457 problem responses | CRLF/null rejection, detail truncation, panic recovery | ## CLI Commands @@ -114,8 +117,11 @@ scion/ | |-- file-upload/ # File upload handler | |-- health/ # Health check probes | |-- mail/ # SMTP email sender +| |-- metrics/ # Prometheus HTTP metrics +| |-- migrations/ # SQL migration runner | |-- middleware/ # HTTP middleware collection | |-- pagination/ # Pagination utilities +| |-- problem/ # Problem Details API errors | |-- ratelimit/ # Rate limiting algorithms | |-- rbac/ # Role-based access control | `-- validation/ # Request validation builder @@ -145,7 +151,7 @@ go run ./cmd/scion doctor --strict Run tests for all registry modules in PowerShell: ```powershell -$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') +$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail','migrations','metrics','problem') foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Pop-Location } ``` diff --git a/README_zh.md b/README_zh.md index aef946d..693656c 100644 --- a/README_zh.md +++ b/README_zh.md @@ -37,7 +37,7 @@ scion diff cache 省略 `--to` 或 `--target` 时，Scion 会使用模块的默认目标目录，例如 `internal/cache`。你仍然可以用 `--to ` 或 `--target ` 覆盖。 -Scion CLI 会复制源码，并写入 `.scion-module.json` 供后续对比使用。它不会自动修改你的 `go.mod`。标记为 `stdlibOnly=false` 的模块，例如 `auth`，需要显式使用 standalone 模式： +Scion CLI 会复制源码，并写入 `.scion-module.json` 供后续对比使用。它不会自动修改你的 `go.mod`。标记为 `stdlibOnly=false` 的模块，例如 `auth` 和 `metrics`，需要显式使用 standalone 模式： ```bash scion add auth --standalone @@ -84,6 +84,9 @@ Get-FileHash .\scion_v0.1.3_windows_amd64.zip -Algorithm SHA256 | [cache](registry/cache/) | 泛型 TTL + LRU 缓存 | 后台清理、goroutine 泄漏防护、最大条目数限制 | | [pagination](registry/pagination/) | Offset/limit + cursor 分页 | cursor base64 校验、负 offset 归零、最大 limit 限制 | | [mail](registry/mail/) | SMTP 邮件 + 模板 | 邮件头注入防护、XSS 转义、附件净化、异步队列 | +| [migrations](registry/migrations/) | SQL 迁移执行器 | 安全文件名、表名白名单、checksum、事务 | +| [metrics](registry/metrics/) | Prometheus HTTP 指标 | 路由基数限制、label 净化、隔离 registry | +| [problem](registry/problem/) | RFC 9457 错误响应 | CRLF/null 拒绝、detail 截断、panic 恢复 | ## CLI 命令 @@ -114,8 +117,11 @@ scion/ | |-- file-upload/ # 文件上传模块 | |-- health/ # 健康检查模块 | |-- mail/ # SMTP 邮件模块 +| |-- metrics/ # Prometheus HTTP 指标 +| |-- migrations/ # SQL 迁移执行器 | |-- middleware/ # HTTP 中间件集合 | |-- pagination/ # 分页工具 +| |-- problem/ # Problem Details API 错误 | |-- ratelimit/ # 限流算法 | |-- rbac/ # 角色权限控制 | `-- validation/ # 请求校验构建器 @@ -145,7 +151,7 @@ go run ./cmd/scion doctor --strict PowerShell 中运行所有 registry 模块测试： ```powershell -$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') +$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail','migrations','metrics','problem') foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Pop-Location } ``` diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index e5f74b3..8fd2c32 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -46,7 +46,10 @@ export default defineConfig({ { text: 'Health', link: '/modules/health' }, { text: 'Cache', link: '/modules/cache' }, { text: 'Pagination', link: '/modules/pagination' }, - { text: 'Mail', link: '/modules/mail' } + { text: 'Mail', link: '/modules/mail' }, + { text: 'Migrations', link: '/modules/migrations' }, + { text: 'Metrics', link: '/modules/metrics' }, + { text: 'Problem Details', link: '/modules/problem' } ] } ] @@ -95,7 +98,10 @@ export default defineConfig({ { text: 'Health 健康检查', link: '/zh/modules/health' }, { text: 'Cache 缓存', link: '/zh/modules/cache' }, { text: 'Pagination 分页', link: '/zh/modules/pagination' }, - { text: 'Mail 邮件', link: '/zh/modules/mail' } + { text: 'Mail 邮件', link: '/zh/modules/mail' }, + { text: 'Migrations 迁移', link: '/zh/modules/migrations' }, + { text: 'Metrics 指标', link: '/zh/modules/metrics' }, + { text: 'Problem Details 错误响应', link: '/zh/modules/problem' } ] } ] diff --git a/docs/contributing.md b/docs/contributing.md index 8c5fa36..7b3e24d 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -53,7 +53,7 @@ Add your module to `registry/index.json`. ## Code Standards - Go 1.22+ with generics -- Standard library only by default; external dependencies require an explicit `stdlibOnly:false` registry entry and security/correctness justification +- Standard library only by default; external dependencies require an explicit `stdlibOnly:false` registry entry and security/correctness/observability justification - `gofmt` formatting - `go vet` must pass - Middleware signature: `func(http.Handler) http.Handler` @@ -66,7 +66,7 @@ Add your module to `registry/index.json`. cd registry//src/go && go test -v ./... # Test all modules -$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') +$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail','migrations','metrics','problem') foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Pop-Location } ``` diff --git a/docs/index.md b/docs/index.md index e897177..d717a69 100644 --- a/docs/index.md +++ b/docs/index.md @@ -65,6 +65,18 @@ features: title: Mail details: SMTP email with templates, header injection prevention, and async queue. link: /modules/mail + - icon: SQL + title: Migrations + details: SQL migration runner with checksums, transactions, and safe file validation. + link: /modules/migrations + - icon: 📈 + title: Metrics + details: Prometheus HTTP metrics with route cardinality limits and label sanitization. + link: /modules/metrics + - icon: ⚠️ + title: Problem Details + details: RFC 9457 API error responses with panic recovery and safe details. + link: /modules/problem --- ## Quick Start diff --git a/docs/modules/index.md b/docs/modules/index.md index e23d752..34b1522 100644 --- a/docs/modules/index.md +++ b/docs/modules/index.md @@ -1,6 +1,6 @@ # Modules Overview -Scion provides 12 production-ready, copy-paste Go modules. Each module is self-contained. Modules are standard-library only by default; declared security exceptions are marked in the registry. +Scion provides 15 production-ready, copy-paste Go modules. Each module is self-contained. Modules are standard-library only by default; declared security and observability exceptions are marked in the registry. ## Available Modules @@ -18,6 +18,9 @@ Scion provides 12 production-ready, copy-paste Go modules. Each module is self-c | [Cache](/modules/cache) | TTL + LRU in-memory cache | Background cleanup, max entries limit | | [Pagination](/modules/pagination) | Offset/cursor pagination | Cursor base64 validation, max limit enforcement | | [Mail](/modules/mail) | SMTP email with templates | Header injection prevention, XSS escaping | +| [Migrations](/modules/migrations) | SQL migration runner | Safe filenames, table whitelist, checksums | +| [Metrics](/modules/metrics) | Prometheus HTTP metrics | Route cardinality limits, label sanitization | +| [Problem Details](/modules/problem) | RFC 9457 API errors | CRLF/null rejection, panic recovery | ## Quick Copy @@ -54,4 +57,4 @@ go test -v ./... ## Dependencies -Modules use only the Go standard library by default. Declared exceptions, such as auth, copy their own `go.mod` in standalone mode. +Modules use only the Go standard library by default. Declared exceptions, such as auth and metrics, copy their own `go.mod` in standalone mode. diff --git a/docs/modules/metrics.md b/docs/modules/metrics.md new file mode 100644 index 0000000..db8b7c7 --- /dev/null +++ b/docs/modules/metrics.md @@ -0,0 +1,40 @@ +# Metrics Module + +Prometheus HTTP metrics for `net/http` services. + +## Features + +- Isolated Prometheus registry +- Request counter, duration histogram, and in-flight gauge +- `/metrics` scrape handler +- `func(http.Handler) http.Handler` middleware +- Optional Go runtime/process collectors + +## Usage + +```go +m, err := metrics.New() +if err != nil { + return err +} +_ = m.RegisterDefaults() + +http.Handle("/metrics", m.Handler()) +http.Handle("/users/", m.Middleware("/users/{id}")(usersHandler)) +``` + +Pass route templates, not raw URLs. + +## Security + +- Rejects CRLF and null bytes in labels +- Normalizes raw URLs and overlong labels +- Caps route-label cardinality and sends overflow to `route="overflow"` + +## Copy + +`metrics` uses Prometheus, so copy it in standalone mode: + +```bash +scion add metrics --standalone --to internal/metrics +``` diff --git a/docs/modules/migrations.md b/docs/modules/migrations.md new file mode 100644 index 0000000..731db9d --- /dev/null +++ b/docs/modules/migrations.md @@ -0,0 +1,40 @@ +# Migrations Module + +Standard-library-only SQL migration runner for `database/sql`. + +## Features + +- Loads `YYYYMMDDHHMMSS_name.up.sql` and optional `.down.sql` files from `fs.FS` +- Records version, name, checksum, and timestamp in `schema_migrations` +- Applies migrations transactionally by default +- Supports `Up`, `Down`, and `Status` +- Includes a status HTTP handler with panic recovery + +## Usage + +```go +m, err := migrations.New(migrationFS, migrations.Options{ + Dir: "migrations", +}) +if err != nil { + return err +} +if err := m.Up(ctx, db); err != nil { + return err +} +``` + +Use `migrations.DollarPlaceholder` for PostgreSQL-style placeholders. + +## Security + +- Rejects CRLF and null bytes in config/file names +- Rejects path traversal in migration directory and names +- Validates schema history table identifiers +- Caps migration count and SQL file size + +## Copy + +```bash +scion add migrations --to internal/migrations +``` diff --git a/docs/modules/problem.md b/docs/modules/problem.md new file mode 100644 index 0000000..28d78f1 --- /dev/null +++ b/docs/modules/problem.md @@ -0,0 +1,40 @@ +# Problem Details Module + +RFC 9457-style API error responses for `net/http`. + +## Features + +- `Problem` and `InvalidParam` JSON types +- `application/problem+json` writer +- Handler adapter for functions returning `error` +- Panic recovery middleware +- Optional safe request ID extension + +## Usage + +```go +http.Handle("/users", problem.Handler(func(w http.ResponseWriter, r *http.Request) error { + return problem.Error(http.StatusNotFound, "User not found", "no user matched the request") +})) +``` + +Validation errors: + +```go +problem.Write(w, r, problem.Validation([]problem.InvalidParam{ + {Detail: "must be a valid email", Pointer: "#/email"}, +})) +``` + +## Security + +- Rejects CRLF and null bytes in response fields +- Truncates long details +- Hides unknown internal errors behind generic 500 responses +- Limits validation error count + +## Copy + +```bash +scion add problem --to internal/problem +``` diff --git a/docs/zh/contributing.md b/docs/zh/contributing.md index 32aa69a..bf7d2ac 100644 --- a/docs/zh/contributing.md +++ b/docs/zh/contributing.md @@ -66,7 +66,7 @@ go test -v ./... cd registry//src/go && go test -v ./... # 测试所有模块 -$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') +$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail','migrations','metrics','problem') foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Pop-Location } ``` diff --git a/docs/zh/index.md b/docs/zh/index.md index 0f55717..e2f0a0a 100644 --- a/docs/zh/index.md +++ b/docs/zh/index.md @@ -65,6 +65,18 @@ features: title: Mail 邮件 details: SMTP 邮件 + 模板，头部注入防护，异步队列。 link: /zh/modules/mail + - icon: SQL + title: Migrations 迁移 + details: SQL 迁移执行器，包含 checksum、事务和安全文件校验。 + link: /zh/modules/migrations + - icon: 📈 + title: Metrics 指标 + details: Prometheus HTTP 指标，包含路由基数限制和 label 净化。 + link: /zh/modules/metrics + - icon: ⚠️ + title: Problem Details 错误响应 + details: RFC 9457 API 错误响应，包含 panic 恢复和安全 detail。 + link: /zh/modules/problem --- ## 快速开始 diff --git a/docs/zh/modules/index.md b/docs/zh/modules/index.md index 92caba4..e25061d 100644 --- a/docs/zh/modules/index.md +++ b/docs/zh/modules/index.md @@ -1,6 +1,6 @@ # 模块概览 -Scion 提供 12 个生产就绪、可复制粘贴的 Go 模块。每个模块自包含。模块默认仅使用标准库；安全例外会在 registry 中显式标记。 +Scion 提供 15 个生产就绪、可复制粘贴的 Go 模块。每个模块自包含。模块默认仅使用标准库；安全和可观测性例外会在 registry 中显式标记。 ## 可用模块 @@ -18,6 +18,9 @@ Scion 提供 12 个生产就绪、可复制粘贴的 Go 模块。每个模块自 | [Cache](/zh/modules/cache) | TTL + LRU 内存缓存 | 后台清理，最大条目限制 | | [Pagination](/zh/modules/pagination) | 偏移/游标分页 | 游标 Base64 校验，最大限制强制 | | [Mail](/zh/modules/mail) | SMTP 邮件 + 模板 | 头部注入防护，XSS 转义 | +| [Migrations](/zh/modules/migrations) | SQL 迁移执行器 | 安全文件名，表名白名单，checksum | +| [Metrics](/zh/modules/metrics) | Prometheus HTTP 指标 | 路由基数限制，label 净化 | +| [Problem Details](/zh/modules/problem) | RFC 9457 API 错误响应 | CRLF/null 拒绝，panic 恢复 | ## 快速复制 @@ -54,4 +57,4 @@ go test -v ./... ## 依赖 -模块默认仅使用 Go 标准库。显式例外（如 auth）会在 standalone 模式下复制自己的 `go.mod`。 +模块默认仅使用 Go 标准库。显式例外（如 auth 和 metrics）会在 standalone 模式下复制自己的 `go.mod`。 diff --git a/docs/zh/modules/metrics.md b/docs/zh/modules/metrics.md new file mode 100644 index 0000000..d6c78d0 --- /dev/null +++ b/docs/zh/modules/metrics.md @@ -0,0 +1,40 @@ +# Metrics 指标模块 + +面向 `net/http` 服务的 Prometheus HTTP 指标模块。 + +## 功能 + +- 独立 Prometheus registry +- 请求计数、耗时直方图、in-flight gauge +- `/metrics` 抓取 handler +- `func(http.Handler) http.Handler` 中间件 +- 可选 Go runtime/process collectors + +## 使用 + +```go +m, err := metrics.New() +if err != nil { + return err +} +_ = m.RegisterDefaults() + +http.Handle("/metrics", m.Handler()) +http.Handle("/users/", m.Middleware("/users/{id}")(usersHandler)) +``` + +传入路由模板，不要传原始 URL。 + +## 安全 + +- 拒绝 label 中的 CRLF 和 null 字节 +- 归一化原始 URL 和超长 label +- 限制 route label 基数，溢出进入 `route="overflow"` + +## 复制 + +`metrics` 使用 Prometheus，需要 standalone 复制： + +```bash +scion add metrics --standalone --to internal/metrics +``` diff --git a/docs/zh/modules/migrations.md b/docs/zh/modules/migrations.md new file mode 100644 index 0000000..2896293 --- /dev/null +++ b/docs/zh/modules/migrations.md @@ -0,0 +1,40 @@ +# Migrations 迁移模块 + +基于 `database/sql` 的标准库 SQL 迁移执行器。 + +## 功能 + +- 从 `fs.FS` 加载 `YYYYMMDDHHMMSS_name.up.sql` 和可选 `.down.sql` +- 在 `schema_migrations` 中记录 version、name、checksum、timestamp +- 默认使用事务执行迁移 +- 支持 `Up`、`Down`、`Status` +- 提供带 panic 恢复的状态 HTTP handler + +## 使用 + +```go +m, err := migrations.New(migrationFS, migrations.Options{ + Dir: "migrations", +}) +if err != nil { + return err +} +if err := m.Up(ctx, db); err != nil { + return err +} +``` + +PostgreSQL 风格占位符使用 `migrations.DollarPlaceholder`。 + +## 安全 + +- 拒绝配置和文件名中的 CRLF / null 字节 +- 拒绝迁移目录和文件名中的路径穿越 +- 校验 schema history 表名 +- 限制迁移数量和 SQL 文件大小 + +## 复制 + +```bash +scion add migrations --to internal/migrations +``` diff --git a/docs/zh/modules/problem.md b/docs/zh/modules/problem.md new file mode 100644 index 0000000..b1d4d05 --- /dev/null +++ b/docs/zh/modules/problem.md @@ -0,0 +1,40 @@ +# Problem Details 错误响应模块 + +面向 `net/http` 的 RFC 9457 风格 API 错误响应。 + +## 功能 + +- `Problem` 和 `InvalidParam` JSON 类型 +- `application/problem+json` 写入器 +- 支持返回 `error` 的 handler 适配器 +- panic 恢复中间件 +- 可选安全 request ID 扩展 + +## 使用 + +```go +http.Handle("/users", problem.Handler(func(w http.ResponseWriter, r *http.Request) error { + return problem.Error(http.StatusNotFound, "User not found", "no user matched the request") +})) +``` + +验证错误： + +```go +problem.Write(w, r, problem.Validation([]problem.InvalidParam{ + {Detail: "must be a valid email", Pointer: "#/email"}, +})) +``` + +## 安全 + +- 拒绝响应字段中的 CRLF 和 null 字节 +- 截断过长 detail +- 未知内部错误统一隐藏为通用 500 +- 限制 validation error 数量 + +## 复制 + +```bash +scion add problem --to internal/problem +``` diff --git a/internal/bundle/manifest.json b/internal/bundle/manifest.json index 2a981eb..b858609 100644 --- a/internal/bundle/manifest.json +++ b/internal/bundle/manifest.json @@ -1,7 +1,7 @@ { "schemaVersion": 1, - "registryVersion": "0.2.0", - "bundleHash": "bea9fed999eeaac57df1837e148a9a8a5ddf11b55dac47740d1aacacb32d1d8d", + "registryVersion": "0.3.0", + "bundleHash": "c2233f0814a7d3a3b651489005fe1ea5e64f839d1469088e11eb06d79b1873e6", "modules": [ { "id": "auth", @@ -38,16 +38,31 @@ "version": "0.1.0", "source": "registry/mail/src/go" }, + { + "id": "metrics", + "version": "0.1.0", + "source": "registry/metrics/src/go" + }, { "id": "middleware", "version": "0.1.0", "source": "registry/middleware/src/go" }, + { + "id": "migrations", + "version": "0.1.0", + "source": "registry/migrations/src/go" + }, { "id": "pagination", "version": "0.1.0", "source": "registry/pagination/src/go" }, + { + "id": "problem", + "version": "0.1.0", + "source": "registry/problem/src/go" + }, { "id": "ratelimit", "version": "0.1.0", @@ -627,8 +642,8 @@ }, { "path": "registry/index.json", - "sha256": "529693ad0317b61ed6c347ee0ed37fb3d4a186716afae194fb31bf76c8510362", - "size": 7464 + "sha256": "8ea07d0722dc2b2a5145a1112f999345068e1ff6cc61ad15c7d9b17403b6365f", + "size": 9372 }, { "path": "registry/mail/README.md", @@ -714,6 +729,97 @@ "sha256": "9c655bb6fd4c7920b782c4f7f757c0dc171d82bc85084f03c44c6d7846882f60", "size": 884 }, + { + "path": "registry/metrics/README.md", + "module": "metrics", + "moduleVersion": "0.1.0", + "sha256": "c0be102354e446cb79732c0a834c8fe4445bee0fa793dbaf5b51530a72768a94", + "size": 1170 + }, + { + "path": "registry/metrics/__llms__.md", + "module": "metrics", + "moduleVersion": "0.1.0", + "sha256": "b2d7789b44374709f96b0389fb3d2a87240974eb7ca2b940e7ceea04086d3106", + "size": 412 + }, + { + "path": "registry/metrics/src/go/config.go", + "module": "metrics", + "moduleVersion": "0.1.0", + "sha256": "79356169be755f339085f9c66d8154c98880b1ae82d203dc498fcc9f0412e10c", + "size": 2377 + }, + { + "path": "registry/metrics/src/go/config_test.go", + "module": "metrics", + "moduleVersion": "0.1.0", + "sha256": "db551bb233f28f99a8e0e781594aca49a335e67659104c7153882a0831187049", + "size": 882 + }, + { + "path": "registry/metrics/src/go/go.mod", + "module": "metrics", + "moduleVersion": "0.1.0", + "sha256": "ec797d2e535d490cc46e277d8ac458ab67b2c7f1f8bc890e9829699de13f4eca", + "size": 552 + }, + { + "path": "registry/metrics/src/go/go.sum", + "module": "metrics", + "moduleVersion": "0.1.0", + "sha256": "eac32fd9b63e472b2a5d7026c99ffce81f2ba199ce37bf8a0382ec0d46b527fd", + "size": 2152 + }, + { + "path": "registry/metrics/src/go/handler.go", + "module": "metrics", + "moduleVersion": "0.1.0", + "sha256": "c95abb449c39906c9ba09ed8ed639ddf310fd3af2264b7acfc2652eade1f51f4", + "size": 345 + }, + { + "path": "registry/metrics/src/go/handler_test.go", + "module": "metrics", + "moduleVersion": "0.1.0", + "sha256": "edd05b933779bdeaa56a3b3680cbaf00b52cfa294e625aa2ccf2318e2c7bf037", + "size": 814 + }, + { + "path": "registry/metrics/src/go/metrics.go", + "module": "metrics", + "moduleVersion": "0.1.0", + "sha256": "a1c38c8ec289bedaef521dd31dede791323d8e9bf8de7e2e8d01ee0e6ea97758", + "size": 4023 + }, + { + "path": "registry/metrics/src/go/metrics_test.go", + "module": "metrics", + "moduleVersion": "0.1.0", + "sha256": "4e16133d8bcf422f34b18b6276dba46fabf771a2b8a3aeaa884f04533fbd7da9", + "size": 1390 + }, + { + "path": "registry/metrics/src/go/middleware.go", + "module": "metrics", + "moduleVersion": "0.1.0", + "sha256": "d4ce42185908998012df5bc45a137e753cc2decc3dfb2372bc3a646e9ff4a582", + "size": 1579 + }, + { + "path": "registry/metrics/src/go/middleware_test.go", + "module": "metrics", + "moduleVersion": "0.1.0", + "sha256": "e9685fa88eb105bef265d205e6fa72ae862eae6e9c5b8569571a6ee665ab910e", + "size": 2446 + }, + { + "path": "registry/metrics/src/go/pentest_test.go", + "module": "metrics", + "moduleVersion": "0.1.0", + "sha256": "7adcbb5fb5c97be5c7918c8d11302340093c4d6b96700b420d9bceb0474675ce", + "size": 1750 + }, { "path": "registry/middleware/README.md", "module": "middleware", @@ -924,6 +1030,90 @@ "sha256": "9409273d4cea1da32fc091dda9d99b3d468191c4ceae93ea29c87dab9f4f90c2", "size": 6411 }, + { + "path": "registry/migrations/README.md", + "module": "migrations", + "moduleVersion": "0.1.0", + "sha256": "a0e9d2d29da22ba6c7af14e0ae93147799f8b9327b4614160be6f9df314d6b44", + "size": 1425 + }, + { + "path": "registry/migrations/__llms__.md", + "module": "migrations", + "moduleVersion": "0.1.0", + "sha256": "1b4f6bbf9f7635143703ecac86db346a5c82d7f99a2c2b9f3ebdb90251137e98", + "size": 437 + }, + { + "path": "registry/migrations/src/go/config.go", + "module": "migrations", + "moduleVersion": "0.1.0", + "sha256": "46157c864be14ec00c6f17fdb33d83df6c83d0ec6d6150abc3b357120aab80d1", + "size": 3625 + }, + { + "path": "registry/migrations/src/go/config_test.go", + "module": "migrations", + "moduleVersion": "0.1.0", + "sha256": "8a8d1b61d16e0bad0a4ea36f375dc8c0966bc1671e13496721df0ddb7e0f1147", + "size": 1656 + }, + { + "path": "registry/migrations/src/go/file.go", + "module": "migrations", + "moduleVersion": "0.1.0", + "sha256": "6e906f0807671d5bc3c46f6167c4c89b807cadeb10e8fda724c1c6d9d242f3f0", + "size": 6503 + }, + { + "path": "registry/migrations/src/go/file_test.go", + "module": "migrations", + "moduleVersion": "0.1.0", + "sha256": "d613b1520f29f75643d7b0134375298ff9f1191beecc4ae41247b57749d306a2", + "size": 1518 + }, + { + "path": "registry/migrations/src/go/go.mod", + "module": "migrations", + "moduleVersion": "0.1.0", + "sha256": "19413797636c3407af2c8fb78b6af93ef4352d97bc7f8ba6712999166da44115", + "size": 27 + }, + { + "path": "registry/migrations/src/go/handler.go", + "module": "migrations", + "moduleVersion": "0.1.0", + "sha256": "2509043edaec6fb5d15ccca50feaf2cbad0d85e5734568c6656112d4618aeea9", + "size": 935 + }, + { + "path": "registry/migrations/src/go/handler_test.go", + "module": "migrations", + "moduleVersion": "0.1.0", + "sha256": "b751f9bca97add4540c010f071615468c13158901aac2a39abc380e317d76d0b", + "size": 1256 + }, + { + "path": "registry/migrations/src/go/migrator.go", + "module": "migrations", + "moduleVersion": "0.1.0", + "sha256": "0abab9df591762fe34d71259a7bea7940f1e3aa94d95bfe0be96f620a84e83e6", + "size": 9106 + }, + { + "path": "registry/migrations/src/go/migrator_test.go", + "module": "migrations", + "moduleVersion": "0.1.0", + "sha256": "8baed6bfb476d3630dc7dafc70417fb22e6c2ec40533f2cab6d1df6c3f721a1b", + "size": 8397 + }, + { + "path": "registry/migrations/src/go/pentest_test.go", + "module": "migrations", + "moduleVersion": "0.1.0", + "sha256": "4a2617c0bcfb8fa3c7378da6ebf967d7337611fca7e38e40541848a919b0c7f4", + "size": 1277 + }, { "path": "registry/pagination/README.md", "module": "pagination", @@ -1022,6 +1212,76 @@ "sha256": "17e6b6d5b0283af8471140aafbbfe3040aff7d16cc1c210ac3033bd0b9ce571c", "size": 709 }, + { + "path": "registry/problem/README.md", + "module": "problem", + "moduleVersion": "0.1.0", + "sha256": "5a7bb54b929203153b361b7a2f4604edb6bb4ed92d51029ed276e4bd2c961681", + "size": 1243 + }, + { + "path": "registry/problem/__llms__.md", + "module": "problem", + "moduleVersion": "0.1.0", + "sha256": "7d6ff6828e1717d67f6a722334ec6428acce33240c49c9ee49f66338badc6062", + "size": 399 + }, + { + "path": "registry/problem/src/go/config.go", + "module": "problem", + "moduleVersion": "0.1.0", + "sha256": "509a046951fdb9d5544ef5f824ad8936e32eb5b3b62f76616cf3e8f2d8f8b10d", + "size": 2183 + }, + { + "path": "registry/problem/src/go/config_test.go", + "module": "problem", + "moduleVersion": "0.1.0", + "sha256": "164cea3db562a8e817fa5c78c76b0a3c7449333d0c42c4e0aee8f65bf53cec29", + "size": 854 + }, + { + "path": "registry/problem/src/go/go.mod", + "module": "problem", + "moduleVersion": "0.1.0", + "sha256": "458885934f9331a46cf3f0c7c72dd7ec2c8f4c249e72848fd8ae50dd7be5ee4e", + "size": 24 + }, + { + "path": "registry/problem/src/go/handler.go", + "module": "problem", + "moduleVersion": "0.1.0", + "sha256": "f7b3790a5080486f4d227a99284bf17d9b54113d9cb876ca57ce68236acb8b85", + "size": 2718 + }, + { + "path": "registry/problem/src/go/handler_test.go", + "module": "problem", + "moduleVersion": "0.1.0", + "sha256": "dea4b9045747cfcd10f0a0236b15f3435470c1f48c291d7ac6d270569120a93f", + "size": 3069 + }, + { + "path": "registry/problem/src/go/pentest_test.go", + "module": "problem", + "moduleVersion": "0.1.0", + "sha256": "c9014ebacd1f9ef7f7dc5e287966e09500b11d67aa72cb8825d0b9fcaaedf4a8", + "size": 1432 + }, + { + "path": "registry/problem/src/go/problem.go", + "module": "problem", + "moduleVersion": "0.1.0", + "sha256": "2712492e8c8ed382a24abca66b87f973ab60956c6be2738f41485c0787ce5401", + "size": 4379 + }, + { + "path": "registry/problem/src/go/problem_test.go", + "module": "problem", + "moduleVersion": "0.1.0", + "sha256": "da8aaa0019189da7fb74309192f06da8eb2f354745c2ce4527695c0b16a28abe", + "size": 1262 + }, { "path": "registry/ratelimit/README.md", "module": "ratelimit", diff --git a/internal/bundle/registry.zip b/internal/bundle/registry.zip index 42de8e36862a2e13a278088f05ba2c1cd3cdbcee..c35617a1ecb568e9c4f6274e7cba35ffcdb338d6 100644 GIT binary patch delta 60010 zcmcG%34GjFbuTW>Xe7)Yh*ij9lct@fFx`!nH0B%F%W#$zL?aBAM{KhO(mzInTaT-*u{0kn^yZazT8oKd|;s0L>+1`*BeVG zS*#(G&L_CxWTu%SdOljShFWQ9iz?UUa=dzQkq6>L{5W> zQe&eu3Grl^@8%LUf7;?L)(>8*{(`l>Mto?uZ<+Dq@2~mZ{W}W^?%%m^_ukE7`u&9~ zKK_HCYnNVpDKyWNJ29wfHO(Z(G)&*Bx=XvvGIuv;3gZ6hiAs4 z_%f{LIjm_XOPw*E#ls`fcphv+g(H2PM{CT)OgwIm%x9v&% z{_mt`M|Q|WX|RteE}CkKj)iC9nRHdPVN7H))4}8DKOU_L=njEMQ>rjkRb8D^39Cw_ z>$px=EP{r!;S@EN<(FfTD}m}N{vrEKRcEV>;o)J!Xo=5;=hNnNIGr}bitIDdscB*e zW_o6H!VIUecXmg^L9;DsCXyN1W;hkZZUlS7nTcTolod9|Xs=KPaE^x2&df}Nfj{Ad zIhu-6uQQ2}yzT~QU6vWmH-QznTjSd3lEl7+^ATTYUjZ8^k^zP9o5Jv?rtpFAPxn* z`z@xE4S2U3Z~N$L3f^2)fPX}T_qCR;u)aV2Z^pXp4!yMs!JHIVI}#q9#43*^owTP> zG052I#4|p8hR2gZAlz^r{eeSh*n(p$V@9IW(L^L|CKKlHIF72B>_~7lIaLSb2Yd*h z>qg@-EPZGkEIl!f%^JiGSimFns~)Z~lhbsfh2!S%Vg$tJVF#)Wn`78kn|dYTp#~#b zR%03;7HM?sFVEiE7kl!zwAtuwfW>A#N1 zzm~`!T;+C2EQwc$D0yw-VE5yKvfCuf63@5z$VV>>?c?fVPCMhlsZe(3W>_8va9 z)4Ngl?+2naY?|;-5mkEm^}K;9a}=ra5*qV1OUg)tNoKUum$XL?)Gt1BG}1lTLk{$xKG- zY<9vyp4mYoGY>@XnHE2&@~+NyQyWY)N`eDO1r%oZ`)5Yd^H|Q5v#={X$Naf7nlOpO z8Ge*>@itMBF5x<9YRDz6GeG(b|2EC@7qAcEOw&H&3NNHBdV^$i*vZUHDv`#qi?i1O zVBk2_5T|Or<>K<4o@K&(ZOK}D*3C3fAUz8Fg%IGkXD&Yw^W<{J-37n|X^c4iBRv*#$idxoD&|iDv5*8(<8-2U8v}gS@)3o=X|4ZRi zEDWh2-Av`CS!;H+5A<~$=^yHD>1yxqZ8_4O`=NiZwf|iIKzmp2hpv{hLw!Ai1MNed z?cMo4qU+;kPuo(Txoy9wYx3P%vuGB9>YL0<=JrEoAYfhwxROB&S;XTF-py7EYkG@Y zSeV2u*6A@^^A_71yk%l_$h%e?eyyimeCypF--=|=v1MX7qoVb^~c6cdxk`JHy<+V)%G|bhE5|+2oiEGZhMQ_7;4Dd z80D@e3YOVL16n}VBU%Od+N1d`=3d8;R&6xj>mE`u64&~&k_FMu*vm`1MpDggF+=tE zU(VvFA(!hW@n{0uq)52hJY)k?GcFoQ{cZTgJ#X}sPvmU#rEq)(qUcyS9;YK-anWk^ zUz!0njd*n!!pI;lkYmv*Zd1*(ldR8b&NJiB=E6PiC^uwH>O*Xg80(kmVNg$mUhk_2 z$O<=^u|0dZzfrCUgKXE$~R$E#>0XO(n0(D?sa1w{(^!IKOeNWHaUc)dp~Pz z6OX^a=eO>6hm9?()VbE3OikgId55?-;@h;hN|j!=m=Er?BjHFbopz9ME$Toqv->O6 zqVPSQih8R)voLu?5C9m?4e?s~7z# zO}(fOj8Qa%3#qtsanI{ZR*T=>?DLB+z0Ff5F1*FFIjh-{#r)}#7@nREfJ>6`Rfu*}Rmh9$2cTI%)c;(mrv|ePD?M=UO+MY33Gu zti53<)9sky;G?aR&~4QqBvu9>Htn+-yXFHOK4IbY_uCy8Gc8* zk!UiN*k3mtO--h$ZhbIh;+&5qA{by4Qqy>rM0F|jK}-auMeF9~Cc^27x=Rh_rH0_% zV0}&tyTN1}m#FDvI#UNhe;SGl9wW3rcyP%kQ>@Fnqi#GIp2^?>JDxQ!)ni)qwGH*5 zgQ0rx!vlNwg_>%^`y-}~Pgg6_j{>auCkO6_?&n`M(zQ@FCAMPuVt zmg;DNKS#%~Qoyot(I9-a2_{qHb#rx)TB%k;!=iqZ$?f)+$dyL6s*Lkrp1@8_4F{>WNVE>G;UWlZo!J(+5UQ+!4K)4DTE5 zIeD?CY2;jL@0szFx6Sn&xGmJQXCmFxb;!6n=3vw4`e&loV-2^ZPM;ZUI5To);Na}+ z#eH?ziPXr+k;J9CPP3Rpn zqz%)iOz<>qnFZ)`^@G+T*rh@A3Iy$c53gbrpYt0{VmCs z=xuG8iM@5V?`=Jr>A9`1@oeYOW8IneJvH6L)OJ_%K zo9IdpHqOLHvPWZ`C*yTzTaR9B4;|~jxF4$61DTd72-B{~=IhWg&(QwJV0^N*Cp>wg zeoxzEs=vSc_9K%Wx6k!AjT}9A$H?Ajd@9;~tmF3NIajxn*`;X_(``E0F*DVhYVQkm z9#3CtJw4MFu1_`2g^$&@ryE046BDfmTTit%T#R*=#~Q}_&Kzt%9*dvJ#P{FU)ISnx?W>=gN%Xgl)%SOg(v-6n8(Xqh z$CMA&)u;N7&2=BgZMkt|B6DWGJK8?h(|_PZ$C=5-1ND2xx-W*uPuKOexM#eGeGy79 z_sU*{ePK06_Rb#cz1Y!FAFAuQ7#a;F23w{M9O&vfK6>iN!Fc0TN4&qQW%{1+?R zT{`93cn808X>DkO=)j>J8l9V-Zta~pHvuuSC)+fitP3amlJR|=`!DVvJUSMc+B4F3 zFtz7EylJl+S+CW(WB$Dz?U@5->l-GHjrJZmab_fXd_Hi0G# z>grB6X3w-=V*#$C0n@+jWdB56DmvM4xaR>*+bGPaVhPds1_!yLwI^zkPOZ{A5@7z@E9Lib&SJDVEsn7gC9 zV`>&>a}Km|jw&r5#&MpHnDV*s_-xP7I|j$kjh#Gs{A^QOYESz3!2Zdpx{0|X`*LXIYJ5C8z2uDDojPVb*1E58zHjhS_CR+!lej$<>zrsk)j88MnAvmY%)Zui^4LV| z_PXfa-qwqgnDSERGiKS>(Xc-`k-2#CNMd%d^Ntgp7bhqSLYPiLO(F)BE-$!*%;_ONH7R_D+SHP7ic;oV&fJ zXW-~-a^Jy;{aq88(U$3K0`na^H{LrM zIX;#dYnm9po<1G3y?CzYcqUyR=^vYp4&E_%bh@W2Rd?pt9qBs;raF)BX^Wjn#Cqc0 zBb{wviFBoU<|E;MnkX$Opntt57hb)0y;yz7YwItc{id-=JoAQ1Xx;{_Q4KxvpJ8sNk_;KIn9(C~~N^>?lnoLzq1@&+>q{nrA z$E7lT`AW6As+NB!GI%Q=K*jwcQW~-=snSXh?nrizTB=Klku3#bj zy|NXyi3hrMaq4ET*_w#XHk-RH;noD>5AJl(!%PLG$rCo2W9>KDMRADBH+4 z-685{NLk&|Lz!eI91n=_{l-SWIg*UbV@m1!ZZ6riygL?GtLaFK7k^#2QQY$lUy)&N zjJWu-;_dlIfA&p9JH_GO7OoQi^#N$`cAhU@yOPlK%0K+U2ZOkO1?ByVP_zO_K-*lG zPK|;Bv>i(n-t1ea?`ErB=C2fA3;8OPpSZ)w536QLn~dNHTMeHP4oog2bAVZ+gx|g zvo*`93%6&hl=!MUap7wv>j5u~?&fYncX>7_n__C7KC=xr?fQH~T5?ZEM-9Ja!*#Yl zuI;&8C6>TFfegBi5stA zMMy1wCavvH(TI5D*20Zh8$b$wYz|^}zyOv+=VW}8hDF@pNDm0;Fz^GW{}NnNN6Flq?J+iN0*|WF?24&5nPLqd6iVWh z@Rf;&e8oi~_P>2*7DYfB{QqD=DYK%m<5)I3#~qR%&|ZDpN1_>?p| zpfj>h=sj+xWpM$6nCeW0C!g!zzDi5IHj? z;pG5Heuxa|2@R#;T>^tHugUL(4RZkn74hiLedXEf1&v>+AqmQ~LFy+#KQGjl0-7d# z1vK3KU|uE%2Bv-|xVYe@f$Dl8l^%*EhCry}@F7?Xq$4w_6pYXX*bJ_}&}AC2Y{sB$}nWQ8S@TZS}AOo`!D-ff#f zFLW|Dn_cn`@LZZ1se!MEFU|UQtS9@$R4@;ZzIeA$eh?qETZ<)d*#NcY$;gsCqEFna zY6+9x*L#+|?7BMuTc_pQy*T>M{l+*5YXEN~hAGYFTLkztj-ejrd80Dm8h5BC%oVDG z9f?ssG=rKLuhF_ytyeYgVZXy@m0CDm2J%4Ay(aN+by4L(EWv5alZ%yC69o&WkSO?mczw_27y2`9rUUvC3yDHPgRplOtKSLQ{^@|ZM_5~^5aMc zkEnUd+BgSq4OG`OFa5c>t}amR96V8c$9D;f(=#o#js_bKa4vN>eWol8(k` zs#!eoxPM~>6Y~+elFoFC`kC& z!ujY-p{-3c|E|Q+G(nNLAxlE2rOB`yiy{7uCR1=jRcAOvQ^>32h6s!sMf89zkE<_| z-RLVF-PYmW48}}Kub|nxl=E>rdmp>pm?1-BpKYh(d=i+$1)BCt-1&811d?EfMfs!1hPiQQYZ!WseHHnNI)jE#EbXW0?t=RQE znog5u24od2y~QWRt(S||9fHWDuNGPiA|(pEY#krkq9h;TFpkf%UO~q>i2B_nmEPGa z2Au9n%Eg_ly`i0iaPiro5(vSSr0W!Jtahmwez@4|L4UYA^8dFLnWu@9_<$C*?NEQ+>5{M_(u5_jw>S$;uF5LL70Zg<6MlN%h5h9l9G z5@N~Ol_dm#aWNF_5{2dBm%B<#QFGqwuVAN9DY!dE5|8XJsal=enRx#DzMbN(10{ZO z>_)f}z3o8BiZzSduD`M5sDJdaZ@zU;aY4bJg~!Uah-cnp>$F3k^jonspzBt#%44iI zn1=g_2A9xe@6B_tPDl>yATbwtR*MIU3WJ41B0N`ID^?G9jp8e`=My-^{QDOk&6KT? z=WwWDFZdd+Ue(p&;pcO--D?zG=^DV9FaHpl@YeTwE3>4`hKu@=O7F{(*&GV=_Vh#5 z&K&!YYmzk*s}2R=4imUyhK8l2XR}c`Jfg$WX^KA>!^KbP&|j9=RR}kxUly-lsDGca zu1!U4bRb?LHX6%Z4VZhD2Bzi_U=e}3h%heJjOX zkNAA|{kphBm_IIF;U`pw%yd~Yu9YAU;=+(2NRSfmYBefE_ZGunA^RdGz>G@r1BrA5 ztg_WKbsL?oRJ{b49UaynPra3DkODJ_+M3@{F67r8}yCr4eB(pJ$%YntxQ17 zjmW|#u6)X~a&=#L7Lh}JB)0LDeF?^NU}+3n%$RPCMGirfTvT7f<)raFr2fm6yPxZqY1CUs)FOCH+Ndoz&UJpa_V& zf9Bun-*nY>-duRMH#hLaZu3E)eTcM^n{ag{& zwaG+$i7;ypDU0CJG6FqV9b%)0ML6c!kR2XN2aom-JL0k><&G0$= zudA!A?fCJouKxZZ@+}U|Ob4kOv(6k2Mqq4}zs6!P`JrV$hh57^RcK%_F2aF1+zv-c zI+jd;-qB08#)|!xb|(^K!R@D}(Z_%t3QmD@gVPi!*M_0+U%DO)WBs_!4AXFQ3FAT7 zXPrOORA>xx)Q@557?f&Cbu61LCc+yyrd&=;o+IkCn-}-&E39&ELbArU!x1t5u5I4X z4m;u;5vs1BRBN+1E;U;RgM4#zY6QTv+tlqQm?8QAHws<f>S&u7%v61Gx|hM}s?p3GqF_d< zZxY{_^lU5TI90fJh)?{xXIs{wxYh9_&1Q&YQG2{@L^oQ)Uvn-rDf*wB6Hf^iCMxNT zBRXy(8OPz)>|9S9EE(c+Q=U~~RnP~gY89UfUlY55IROJFS#daMl{Q^Po_6VZnieL& z5uD^psI`!a!0OD(KA_@Wxm_})y0P)DpAb(YoidZU(R?JCoSclpJ4(mUs@tsUijfZ3 z?M!E2MZr38jRRrdxZ{hyd!aYD*C#Gc`>UPbDG+uo0%7$ERfMa!dw*d?dA>9Y;@OCQ zXV$K-!&GSl$KcUjqt&#R9atlG=n93eLL~xPelP-8RPn7!xsNX;m`C@@$NU2 z>~?OB72)cDHyN5_kZagXBl0FrpvuR*HH)_QmVLTpt_))0*9?TUaKfmtf??sclZ5+? zKR_YD;)#&&h7FDdLL!xsp<$v6asM|xL*lE)jItXoe1RCM4h80{gkcU;7f)4rH)nx3 zWY#1DUIeEGqQk-WV~I3Ed$~|SAY$@~gCdGiSH=^^YW+}E(e^=0r7wFXIBOsn2pWDG zLmFNTsy9HVN;k3iSUSstz*Yeigrdt$mo^emEPbZ-;tWt_jPedJVsK|V^gb{>GlI)m zPXz{tokqi1n7Qn(X@{5jY!7|QNg`{EY3ZeW}JI*VmBcf zhj5P|KnNZ@sYy~q&BMGnCrN%0#gnt8?8YK`mC!`a8gb`-=Lqz-xa&Kf5;%wfw%y5D zByu1JC&1DYXr#_E2QgF0*&IY?O&)r{5Y&&lj&tb%WFs;l(e^1lh()@%2+gFTH>y}} z1cMW=h$q3abJFfGZHd^Oo-GvSU6v&w!Tuk}jNDc%)Rx~6{7p%8oU}l-TzJ|{ajvRV z;~TXOnK$1IA)LNAKt^9qQJNF+UB7qbA^M8dI%lHcfKQ8KzQa(3TWBNFjlLVN9RS{O ztgmIDqo=!nsI8-qV8~(NcIiM%Yo{F*ZkJP_-_efF_WnF==o=P0)Ov1wAa8}Xj(+NP zXrQm9yT9cKkDsTX-cCfMAMfdGYwsKCA2`?9ZdL`@{)EFWKr|AF@Tr9RG{updUH0^_ zn8hLC;!6$QZF?NGcyWvCS^LbPP9)~fTxey@_T-Qovs#5mufd>EJ-J6U;m{ zT$N0dy-xYs-Ys;h?gBssaiU!s%yP1nta{}7CUsMCh2WY^}9dD3*VV`of~kH{|I`xfdi^RVEJt@s!_N{_&On?&&RLxA2pX z_rAJ%WkEsp%D3EGyvNRHW(9*gZm`du@b4DWZ}yayb~~cME4Nx^d5x-Y;xMP=e=sAk2;JcK*Et*QQGTVc%Z(nlCzO|WqLJo+c~{YUYwqF0 z5K!k0Oq&E`3nx_G6+qY0?&V;_Nf$=AA!!AkyRlaCvf&0-1oLhzz*+tsYxLL>D^DG7>IXgiV-tP^+L;88{DL&BU6?qOvAhrM0zsKB$73 zVB>;LB5;hEv*D*0R%pfanzP|_lG$L2@QKFSXWa;xPx2kbw2X%D_s&5bItO*=?sYZ+ z>VOuAt|IhzCF}OtpA6>}cQiFWok`ZEC&CSlO?2CZZ$y&vnIZINPCWder>xQ+h$ZXB ztgA3pN+V?uU5VF=udFNfXAzk)<+#Gp?XnZP56(Sd_?7aqErytK9dm(C`&1B@bq3N+ zIINWvGt+gXAk7iPN&TeneS9V6fRk+0p(BBqf&c?LdrnKGDJOiQ8ysJHNo?1*;lXKO zy|aMyq+FzH98ozobrX4!=)^2=i6DWR?s=E8mJY=Rvb>v;ng~H4q}tNA4{!nL%}Kqm z9!XyAmFHhDs1=fk>NtPF0uE!G9oRtvnbo6sx$qU+Q63wBCKiI?ZyWIm2`j{NFM2mk z5+tq8(@LL(%C@&@o*IssYTeh;vxMBp%ReN-4;57`4D~N>x53jazd2mG4L|O$DW``U zHnPFt{;P`nK3ufE!`U0{2B}cs_cja4hakZ;!rmkB?o?eX??!S;wb#D8 zK|KEf&lZQ`2}~oI#HBYeuhb1>2GWANt*DVls^K4**O*3bUpm-@)dm-KTjO9p>1Z@@ zl+-k|d-Vof=u##eR%vqc<}Q>;-#j&07N1DxOrLQs6C7X~M5dc8*(0bA`HAqFJxj2D z@F}o-`W674`qP#xuk_{BGW_vyx_@SDEH+m~UE#D-mI>))>$f{XKQq(R6T1Sd=C>jU zGPk|9X5{OEzEK|$c>!-0cYnILtlomgo;~=6g<1vD2X)adJ@jE5G<6^G_#@V4!28bH zDyvX#8to2S*%kYdwiAmkNUI&SHitndL7~@v-=D!K{m6%XtLkBI7=X`?)s%3}Y6*5> zV`q&?eTc$8E7?j-S94k}3q8)OnOu;|bvaZa!ioZ{G?HCb7}acAv!^LGv+9j~oEG*bHq@sdDG@#v#PTc<1#0dA^ea~HCBAl)GpJ{)OAtcn^S*hg24 z!o309%Gz7^RutT$UH`FF|iu+cIkyiibkHYx|7-6!%bX| zZQg9dJAM9ks~iJiuthDcYaM%=Swy{?EtfEoRU)rl|K9t;=eq`XIZe zawE&Jp8V>uGXZ(7BvaCy=rG9PFauX-vaz-EO0pcC2&a&l8-SI^g+I-FmpL?UetRTI z!G#rEdk^?9kQc0yb6f&PG`}85#im>gTlNBofbXz z*IyLKUgzjGA}%$!A|W-bJ{47T8BFOAa}OKQu0IuoXO5%ouEpxBUwltp40(BS*-qmZ*{j5?^v?ig#(KPyBtUYZJ3O! zhz4kgYKMr2vw$r>;xEs8Ht%-!xr^*JqtPg7#F3hpM1C>~fx!9vh7bD&@h|TvS+Rn8 ztTAsSN1+?5xu2$-el+92&BZc}s(vI{$kRxCWv{n#gCu7)QKnsWR*Ac+3j;2ujVXdz zBR(QC@9Iu%pK9l3e=s=kivW&8P;n4vAQ7@PGxG6`4}sBBxcK3wF*3hA14n3izE)Q+ zyz9=*+fem<^9B1#pqxVJLRN|ZSB5ItC>jv1xUW71oEQpLAZc_>^I&@k$t~s z`F8$(H9#=8oP)FH_mG2=04Vcu*2;XG;!pm_Q&#StraTnIgZFy2 ziB%u+6jx}Bvh+Dsq>2%b_ZL+Z@n1LJx33sWgSus$wc?17R1fLgzhZnVyC$s~m z3Sq!SG$0ic%Rxf#^i~-Br;U*=`sZ)gb?%G8$NcMyd2vw@d)DK0gTft#)K9V>lEw9v zMm+JX=j9dommxZy^Bi31p0>Dku&7Bq`0YaD&zPlf^fBDIBu)M;{dOSdE<{mInA zz0a3t`Nj{@lPz(Dn?Bl4>MZDB8=8>Pk!>9H4MX8bWQfv5ruD^>%$S$kz;Bz8GJ%{g zs{%**+FJ(NP1YEjT)7I4y{#R`I=Tm{ZwgdjAq%LxJM^n=cR^LG~F zS+dXV_0F=9YcHNABn*2Stz%GYfW%`Dc}*|$Eenr+w|oU1WmaWz>-NG59FwF>V@GJ$ z*EC%a&^Hj+YLPx)e}Pjvvu2Ut%yVKRgQHR?rxL?gnhmmunj)XFU3(}ykliNVSm?&O z@bJwKT)=KYQ?V~;*3Hn?ZR*R2)db>Yv@9N>R)*fZG`Op1+X9YG3T04|jK-LmW_2=6d8L|ihm@ej z)*pE{UiXZVQ8-9E)W*OZD(P?UY(Fw!hOT3pG;?K|6FpnJ$XSmWi<|%4C@m$qk%n0` z=}u&0g9Dd%;ERR73z((DE1vvWV%80N_chux&jVfC$*wEB+_F+MoBPDWpCES(tYDc# zh7%VJZ<+Dap82mm124d5Wb`1trQl?7lx%mqg>sI3yQEMbxb)%^n+ggFHi^(--xfiU zht9TEDzlx=4)g;GP>pMc9`Y9si040Fv@&z@wS6>eRqbv$aT2_n&qffDb5c7jawCD@5so)sdrY2cb)Y5#Rq=t zUuAdO40KMW=rt*8Cp55T65&gDUWras#Og;zYi)@d@Pv}_J+LEK1t z-Br#~7nu8`L!o%?d4EMVGNK%ZlhC>xg-*GxwTgm3Upk{(iUM7yYwoeN$)-u$pIZ67 z@@cguksHm<`Pj;)g03|Ib*ja^#YU3@D4ZOwl(299(5|%DT4d}HFCH%K7XeYUO}y}8 zQMuS!VyxLX98h=EK(kAD2*C?~QNFl!u3`HWQHbq3jmm893P`3|N-}XC5x*1UG~lpT zpb+A?;<-2Z*DV3B@?3aPsNo`jaX!1%zbRfX?nAtX*m1tNj2#yqZGQ6qeIKv>e4k(a z5tPZ^AuVbNkElc3a|EH37M-NU6Cd%c7dOAxvs!#`t=ACdGOsV|u=9~d4{E3YIh5Iv z)g(KDZ$J~sUV3zGw!36g7;yv&xt3L68Uw3=W{U%9;KYjy>QtCG7@oRfS63E%RnoX@ zmtsG3XH_OVMt4>`NRztOJV-|lvxR43i=)zl2>(FI4atX8j+uABoYq--O-m1aT2357 zsdE49CY=rcPhR2;za`jEGf2yZUv=1$AvJZd8!8Ql_n|~k@Jz|zX>QWhkX1IYBW7)1 z&@}mTvNvMbhOh>v0+;G9-<0J%<;>;Z!?&lCiDrwSh8@OGRoP67jeqMm#0`JzUzJr0 z(VtBA5mmobS78bI0J^f5t*Y#1KSWx)>8?E#&eSCFK%Z1=ps5W*)s#Ugw*IZZIIEfh zXIQC`zahlHsW4@$Vdb*yB&XpJL^7`9?m-EVYFzv$r-xmDmlEb&4WY>AAZ>E|Maqr; zv&{SQtTYRQKiY@8+Bc939nWm-44NHdoLq&_l)wO|YiD)^Dh4Bln>1gx6uWgQ-qNPa zJgkh1$<@%9@L|X*YiQnb(*9kCLAac_6;qpSu3c>tw=~y^Ct=97$n-K*mmj3dVtv5QY05%vMMR4%aaMIbbhUVA z+^DGH@#%2GGUTG??GP8ool!hJ>)R-9KjvGu&@r$~On<;r6b4? z#z5e`+@V|81g6?bvmHKUv3G}DUw8q8<1qq8{Z07q&9V#psa`1};Q=4*i2zI%E^5lm zfg4eu5*QR|ZU=3O|H$~(-)^-9im6U5E~iZI2~zM<(1BH#jcm}pI68(r$#)kl+N$PO zy@;nXz74MNG&i-!VYIc{FFAsR%{1eAqA9(zSEua@plwUek>OE9+EY>J5b$OqEwWaN zI~p3b`ofHz01L>=^ktt+*N)35yB^6;&hrlt0xjFe*h@VSD7EHGB-9tZ<=1r>d)fc=G(yMSIV?17{TWSCa(p+7bL15=hhZQ{u+VHy!Xx!dF; zd7XC>nf|WtD6(d0ISJ?z#EkL^XgKYTmvscJSeU5EhcvqEr{|~y(af$#VmMc7k&=X? zUUJg;p(SH|Qh;fp)JC&GE!?(bIO(n>f?CxbqiOM!JJvrSII=hiac7d=Dx*&IH`2M< zlhvivoYWA|J&YyySPV%<3LB!tmc~ZS`)#beZ?(7b_l1cs{~y9c4KE89 zn;f|D5@5#YxSJ9{FM^T_9W|8lp=GaQXoZ&640c7XwOdQz<$4!hI<7U$BB7w>(B~+O z1z-kr`!@WDzJwz)F#`CI^-n`7mKm!Wb}5=Wj+)&)17`c#j{bpuy!C-cEi_(9!>-xe z*U{C|cg{T7ey+wykk;*VOW%>>EqzsZ$g-MScMo=EJ8O(l?VPVGH0`s#<1;Y?M5{yR zWDa!T0po#|u3p`|)1Z6{NNq!bSgaJFqYxPA;zju=hS!LkA4T1T2!03}%c~r~mr&9= zM^-A~@j?OqpZZtLvCX-VqO)Cz_oA)SCDz961GGb= zGPo{Ks~K8SLY)Ku<=U(nSiHwVK(GoME|o4aP`C)q83Sd|mcek(qt`ylLsP=Ai;PkDjMGG6BhRZJ**U6iz@Ijk;$Mu$rhhsjkIvPDh% ze&3K$2Kj&f((6XwyrZDt%{w?<^w8(LD>m9-a#}*kf+;BJdp>JumH7GR!TDF7E?VP{ zCCLDaTdH{X!J-XWX~k!oXpTYOVI?J0KrG}K+3Pt~OAnF@Q`(f#ApIr3^plz^-9>Lw z@s}>7kC}6R!UuZVi+G;2EMC|>wi{h?L%b*eZ#=d@(J)*F&6&1a;;Acs;VP@Qik@Fu zigzO94mVR7lBwS*qKU-fJqenF99>^qs2+GQ(Z0M9knM&xz@Cj+rtJrk!_dMMqG#9OEH1<+J(d~g69S@CAXz*wtbUYrTi^GMOuTDsx7@j$%d zeV)xn5UmV@xPyj=s1}&4U76nhV|@{^>|AjgF~;J=>wKHCs6%=Wu5g1jiOXu=kr8gm zZNGd4bZ@l)CI&l`qmz2V5JVs9S!9+Y+L@h+`L!OGwOD+k0h=+qneoHC( zmlH5~l~B)$i?dSHkgiBo_ROU~3qZIV(DdYb2#|^= zkwcAK3lLW7e;DGT_0K&aJddDA$q5a40J%QdU0blXze|`g2a{iI9rOwHT8du3vN5T- z@%9mgT;_K45DGTdwxr7Gj!Ik;HGko06gPgvv!)u^dFhpQ&UH)KkvRoaG=k2rs3qz) zcmIW_T>Qb?iz>E3!DHrprID-6R?_fPiHAPmsc2MYY?CK8FsG-5{9g9N&`a(hAMhL# zk1Ujw9KKpRVi>Lk$>E(!pCR%5GViXH7TwbM7!d;>_H56pDFml%q?Oy`_+W1m&Jta& zxEl1c2v3C8R`NE-Q#34;KtPgf49fb_n54D|pu6vbo|R%mrG$>=|i10MH6x*>B z)}ZJXDPUT1meUA8`d}@Yq8F#ipNE25F*EC+0vg{Em8w2FgJ;>~j2iM*Qv;LY+UbXL znM4QZvS}%Mt%jED0VrX}7}O=CsAk`Cuu0869PrHImU~?Qb==M*KotYaff=sh4(M9I zCFm-|a}18LvvP19OzqeNd74{-o*svbYfOo3OZ6%{btL^)tV2@gDTjFPsbcyxR}bNA zRVS;Y&`*BYv#mytdOZ{>etaHoXrHVqHtMfIB0zn0p*h{&*N=>3;@J-%&FTyuQ;<)Y z$mJ;#RH)^&oAI%3McC>|Dzw!XJO{>0=Zu_se5J5DNA@MOCo_yLa)wL;c^oDVy9V*T zMHX<)aCJ2q0criZ{enqH*(H)+oQuj@8^F|JEl)Sp>%zre71@_pU5D3%>i~VpS*}IR zKpHi+g4aU$F^3 zFiP;rmwW8&PLWZ=`eYwGR;g6SruXcDs()h<5wh6&A*j#gUulINJ!67-X&KTG!J z9xJjbIci@0zl{}Z>=rSYl8XQy)0)R1kIMQF$5b%?LNfA3ngVC zW+PQ8bpmKnW&j`cjL=I1f2qt-?(E*eZA(lP|* zRc&5zJ*CC(eic!61JUvcPo)RHGo5V)j zh0#ta;;xG{S>T|EqOe27YD_hoEF`S}QbGpk!bqu%em6ADiT!aIglr^r;V1?KUaeHx zGlf_(PA=hi6wZwtEnDE2NfAK@^;LD-_4)`ld-Wj4B?xv=-;O9pC&}i7Ov;GK*;r&es-rQpaYWTD>dv)os{Z0g$E)^! zUF>BxX7XIF!eOq32o5g6Jj6Feu5#k=Cp_zYqzDL!mUk3w@d5*gkqkmSk{-8Lp{WM@ z2~j(Ik)D#n;b&g?M;7LR3d_hOG|p4cMvJ*+H6BH1oBKsPb6pJNEIK|0U;dqlP6F@ zOyQNSO=<^7K!tXPZocU*mCNqhf+kISNpAGJNDRpY-MlPL07j6#1|Rs{xK3s^=^kff zDS;$G7e;qVfY1=8`4Xb1uO`lFSaVv{+=pMq^!g&BWD$`IODd^V`_9%vPM#{STUs`u z=;QCL2>kd}#irY;UJW zdXSj&>Z8|Rdvr>mnr{M?{JsyR{#;@vYt>AS^mQKXNU%qbc;X&ECyi~r+PIeU57xdY zku1Gn%eiE2P>7drH&%+Gzbjfx04P(#Qmh~48tQ-#-Tzdm{7v$?GE!(r?r*zmICBxu z33cd@II+dMnq=V>rI?yY2{o);+o&czkW6+V$k|fRA{~=hGwiwEBpwh$ua~THNU~vQ znV5;Liw*cvWA`3YD{=pIo^nFVa=F2zZSVZL)x z?DK#Aq4U2*5bke_#KTYdtHts=OO}c6JmoJJwRe^nAAfAV$^L2RC z1yN{If?g;)(JOyg&y)2XJz^f**SKH3ox!gkwe)t-TR7ARvSE4}0ku#A^4mWAkg`+- zi=mijQ+8N&ILz_5!z$-QFOtwr4RZzolWfQI+So9^7CWq-g5q;$m^suaC(qN<+(V(G z^jz4mN??GVBjIU!n3lo}R2~9)um#6iDl5$o!>n(iR=xD3C1x$3M*}b;k$7|#Rx5n# zi$W(2`MIMF#uPjxK%v6?2neE&@tJ5DY8s8d9PlunnUIf2L5HN?C#&VLlH7?Be$Z<+AbSDPN*+5UDD#&hSHM;^Cha z1-Hvd$VbGK7E{-2C;!N*g{M0!vV%}eSWhfzF!vYJ`Z%c*c z18_({hH}ia7dhRq<*2cvPL_cWU8xqUgTAq5MD~O^e3M@j(-Rs`4x6tqdllX?)-g_| zff1S59gJk+d}GmuZSqx4<;kTn3JXz4Bfj-*|4wlz;N5R+AiwqXrmy|!j&FeSzQN(| z_qCR;u)fn_f2nooSHGO_Rf{4Y(qXx~ zQY5U_<{O@yJzUlop2vDSaKMtjTBH(-xB+$YyaFU4XYlIbLI&!mymdTDZW0unDp zfuiep=sG!c9ehghQr2j+PNC9JEK+Avj9MnsZw`^+wt`y+sLT}Auk$Rwfp;BmhbbaM zX0^%iID)iDR{4VspZbe8;We)}6|&O}Wm&N{K)>g;T<<^caoWs*xy9Zj`B=;?Xuv*7 zBk3(PteoZIpF+MZ2jK&QSTiNMaP}@J=;-Dw0cbjrjDdxBI>ELwVfpUX03gq!xp?UTEPEumU7YNh^R>_$LaL^t?a^2?9!#ipbwp8Z zygaLKydg{uzMSrijty7$GChsl31sx1r6=@q`{eGhab%~(kj{>&U$qy|^--kpqBoyt zJzJIWRKEBmUs5m~^sirfTAOw5Ch5eG5VT<*BrtmqxD+4u>W7^4M#v9Hu1M8$CJ8=z zwpNwZcC?X?b>0V?uJ4(q-o75Zm((>haIUv~sI{fvdXY(gpcgUP+6P)XI)^&jy9qAz zj+66idtYBqpY>Xk{?gHXq;s&Xwtc9t{S=(T2ZlP@avR%UhK{$lz<-nC%R2GUOX+L03-a~=h<0D^Y> zD3Dd4?+b3YA#LJ;(7^^a|K@x-#9v&^B+RloV=6AB29&?@G{aCJ1A8Q^8A@H%HRe=k9zJYrS)~p!{qBgkIO5y1%<+tnD147+ zL%+3~Y?q?-LA}S}EB0vsi`Cu+t2}oF!-6A`J4V(JfJl!76o;-5UH;WqQ(bqWZ7lNB4Bz%C2mu!Kk_W_`r5G zdbP=ufSq45VxoDv!Xx^B&5u20h9>oBP(1!zVFf(bP+w*@CwEt=Mu`TsfHpwVJ0pX9 z!9bCAP6a2`J07H`CnBoss{O1sCr?b}?TCr*uohlz4kie4PA3yJaXgm7Uhm80Y^ejB z*St=16khmH-sM7=uigs8kkuf|BEcVRgeAplxmaaakWiC%7Lw@j;xAuZ@De8710^)G z;|@0t?U-Y&;IaZ*vq{=Db^UR7Q>{fBzB&O@kobuvrDStb=H)0g)b!+4YoYI<8+PI@ zt5FL^Iz;QaG9Y2QQI?bKh>F0j=>w>|Jd68oj&Mss`?SsE0YVDnX`9=lVG1tXlp~QI zH{>+tIYcQDXBsv2LKG;frg@6^KC8OGTR;8c%l+Q6EpT9sg^AzrS6-kTrD_^gYR`Lb zn4D&0!Cvc~98D(Rq=h9*-~(P!A}e;gPcDKTVFkTPq|!mkOppqo)-4nc*yMh;@W7#U z;?P%pD_6@dZP1E8t|>Mb{`l>!tK#etc7w(oQNO#S(mRU`iK{~;G=hXhst0saT#lmCxnz=>?f_*D_TDgJA9BcAT^u3H6= zuvY?7VbFl$i5Xwn3Un5q4bGA(2w9<8e(PU1jK~ZkR$x9c5XC=e3Pg4c)?iDz3V$K+ zMpcw%8^3!Z>YL^n)}rovs&HNQT2rQ#W8Kg+N~*SQ|7wdV@2A#}Nrjq)11hs8U%$qq ziD(L1wZ{5-jRwRcQrorqKyAC#J)tpEju@^it`YSg@!yeme?7BTM9ZrSH?3K;wc^?D z7H?|2p7rUiRU3>TfivtS{KV(p zX%rN^)8OOtsrNXJPfF-%iEFD~hP0-;|EOfOxM#Q|w47(8XkTpm*V2um_bbJlHAb;<}Swuz!j zG16GPmMazmF@cxj%RemMAX?w+t=vl4IicL51(1ps`lCRzUQJ+F-1}31`+RJ=nx^l!e^Vy?)zLELq`xJ3NoO+{NubwhFQ zWYGpcJ=*O^OIqhMmh-$9~Z9>e{xf?nGcSR zIU_yk79I&n(b|&HG_ad7zhw+_lD@U@s}x~x;rhH|2MdQpc&@lstR6s$>at%HmoGi) zi$6l%ZFo=^!VRh0^s!_1M^jjcNbxS3~#%bh4rmVzx6L*{PAVI@{5Axk{}ec#sZmeJ8DOe+yODEh2Fa* zClPPsgLq?&aIISt#_t7a0de=w{9FCyZ}{^c{B{{^WecI#Y!NO-`|zsLYIWH_*5p=E z_cH%-aphB<4J$elipSDDej!{}EyS^Kth#gZ$##-@xn7`Emh%;WzxA^KkB?%_cMB_f(W$(GQ}uvkd1R24#&T`} zw_{eiL%CN+IZo4cAsPz)-E8Y)Klq^Hq|U22dY;E&{oFONy+x zB>my|kh)jk<3(^hYAos>^ldCr-H0`RECFdXvXLe22?efkTx&55B zY)=;F%SNRn3ihq?k~xh^HRis0U@%?$IMbetrBF?1d*q)KHuow+FYFu;zT0I@Iw|+&@KQBl)Wb9_ zFD!0SM7_VIuy8&64wRm#YbKp(jpDgjarb`~mXRpX3=3rO@&)rJMIuoNqws+@7nggn zu&YP|+izLl2uvjTfR0@{%?J{Xg)Aitw5;@0GT|QRk7jC*B$Jad=tpu^r0_`fZsIi! zd)Fn28lg|T_I-bf)cJoZQ|FUbphB@p9Y(EwiqLxs%e8C-n%yF@UoA02-8W06pmqmB+9g;M=Ve2XF^XkEUWHz;rn70NWWoG(@BwLp`Y)mIiHQn^ejLt1mn> zHbpmCrC^Y#rGuJvVq%H>l*L%}qBRLBiSV?vd*)^UhP17?4Ox)LhC#{s;msAEj$#30 zu|yR4oJZy@dzU#KpFzMf86PZriDi??fLmGp*)t|4#wrj!C;7wFbEjrBbE&qVk4g(7 z>us|u5}Tu%lBdyU2neF;i$z)wjn>9U!%8*Z8?d0_e0=BI((KjWFc?O|_NoQ$Cn6=>WM<}i4DG&tb<&{I- zrEDz+!7i`D1o{$<6ku}d_*Ny(Lka$}$idJ&5)UUPv#xVUYGJ@2_2C0J!-td%NOGT$ zc$PIZ4ttT!^&D&6*U;cJAS#NLE+f3!+Blwvh?^1kYe3{?_%?I}-iEks)K`k)Y6$i=kMg~xB(ot4OfBa_*)W4V@KXX(PsJlA1)d5%E( zWN!<41%r&eI!2nuq${(8re418yQ;eTrSGI`jgCtS=>>&(SCP4l7N<5=gdZ;6c1SOa z@1+{^6v9;H>b=xu(Aom=(C16GG)V-ZYZUZIj4W3CIrz=ZIEeX`cUA2cv+>|ToC#vO z2Xgb&!-z29^OW(9Iw1x{VpaqvBeNBy-@x(7#jr2qV28>?wxCB~)vNBTIz7ZX7Bvx8 zh{}QKx|6?IR<#SzrBLoRa%`aT(&PgMqG3fGl4gqk$cb{CFi%`ce>_bG+==_hDN!M4 zDhs^hs?L>5TC7PvxYYw_+)#=SYA2jrOKy?ueBqvV zlpC@-btJY&jP=>Yl}RNBflKv!i)Fgfi(pA*$)V6%jz$>gV4p5MJ=b955v!l_tc&n( zNKizKuvb$tggFr+;@u8CJ?>*yg`4H-&L&>0<5xq5UHn%CMD6ZUXlxC&(IT*hFhU*No0r|luPdPXfr~Jf*l{CH0 zEpvPbnLa(|0lQ z9sDSv<~w#q5>0$NuF12PWS}|pl&BcZd%}s9!J-=R=$*#$N~w6kUXg3#_QwykY+8dy z;MQ0sfu(N2luOKU3*I0$^^`0Z&;6uunRs-q*C!sm(NiQ|QRgY$q$E4-lGe}S8p8P? zuZWg!A$sVkpOpBTwM?PqK)QG|)PsuX{qX>)0(1u|TcsuWiNfC$@7MxvrDQkKY2*_` z#mN1}hBf*=1dR`eLQjh|`#kF_A&je$`=y_5G);ug`fi?J_OzuAf>W`JG(dzFd9dXm zmLi!xzw@xOE_58EDU(XpoCciVfwMTszO&?}j0SiS6c@2*%2nN}OjNjpiB<{Hmn|07 zY&IyBkQ5~eZUKk8th&RH^Z<5gKx&h$&kf%f?69$YVwlxd$W8O{K z8dJ&00mOhngO`{`o(wG+>Z>Zr9q52C=&Ei%BzIW?+4+Zx9I z)Qu&06z;%rymC+9ctrf6h9e&Ee_!{ChG*bj{tUa9zoxbHHLd=SwHp8D?Z%J4zvg@Q z?<^>|f2Vlz-;LY6O$|Qrt=+!uBKaT2Y7u|Z*y!21U;N>dMz{AH2YlkxVPA!K_DSPr z@BSn9uVV48&l=ms-fIIH^G@ZUz2_##y}oAimo315|1`Lway`@);~*JkmVH~7|whdzsL=AK5a_k2h- z{Ls_JE#8|yrhfhULEi?h6@1%AUsLdAES~;R@QVK5d+%QUNne2#$^x%y7^0D~^xupf z!DBl<^X>cL4^IC!yV^Ya={#-VqrWTo{O)yQ9q69^ZFjYK^3gnP?)tW|JNWni_^VH) z;O<2KZg929{zINNaJ#DuzSkUUt@0EU(7&y&Hr-#&(*|BsmAnzbXFd}CrwRD;(LZ;? ze|&s#!-WU;R*Khu*;pMc|4?M|8~{Q80dfAGN|7DfMVbhT;z zMxHjXO~V$4Jw0?~_`T~szN(;r{<#PH`zIGS)OuXKdEw6+n-7)0`<0(dK<`Wc++%+o zFRA8kcy1RD|Ff|@IQrN(-@2!`pn(4Ea!vowezv#^Vg6moR>th$|El`rpS%$WK>v2T z8dm>0Ps7kB{jlX?-N8*KzOw!67>)k9C%xWRylB#5&3_y9V$XkoJUf?*mLDQ9;f4P& zZuY!=jr_6U6F7t4@paJ1*Qm~W%BXmYSo!~qTRkP~tsj0d^Z$&SJfGPmOTI!s0zvDC zPXxb>k^^nBH>>r()n2k)!b)lrEc;mG4;t%l*s0iQ+LT z7ICY%L|pk5xW(tG_-6?&zVEw6vnT#){dbYL@Li1ii}%PjmroS0k>mc$chP3^gR*$d z_l(1yuY6MenEsw|(9`-U`Qr=UGY)y4|FZmXX$A&+o?qwp(fSX6Ad6Srql^ENir-ZR zH#%XGD{m3}r*?PD9ap^}`!p~ntKX!_L{SoG~e1U!p zdG@}a{<~G&^JBp5Q-5RqC=$>7*w`L=38IhwL}tb9uwX=khkwv`r5+3E~Y0?dOzp$t+})<^zxJf*zM&#~U58o>j(4;%4+$ zR_EWE7u$9@UC>Q^~ITZY`B8B4|ej=1MQMwRr+@ zS$eUf*>a_qa}9agu=HX(`{yF{P5bh+Vd=$o*3QMOjfe8IVd=$k)dumng^({D&eO)C zoS^O6mP`E2EqNNU6k<7Pz){0}sD?W8w6SO=Xq%368~)MRJPqkIK4f3Cb0_!*FJIh7 z6WO$is9?#(meX^G{YCQHT@ZzdT|WgDwa$op`%Bk}%GOdvDsxo)y?I&uzmJxdi${KH z?DqWhGL=+`e>+~fTx|JIlsxeoDmf^Rv^W1JT5Wj)m24LO@Siwh->7~}-(G5n;olmY z#j&5E&Y!$P)V=i zDjWYZ)%uq)EPjOfJk%nzfk=9*ydBJG%?)Wde=vME8zvf>Rg0DftzeW$M%9n{#S!11p@%=TG ze`)WbA%0z8P=h;ivq9q4#RzfM>9P($NpO4X q7f5nl8CX(vy3K!}yC!S`^UTBoyn(G+Ah-mCKRg*2(!K(-C<6eFi5=Vk delta 1828 zcmYk6e@s(X6vxYZwD0x3i_jLZmj2?GgBCF@4`lK$uza2z+N`x}U%r-4W}uJsIUmbVP-udFrVPwAR@IBF&a_$Qj!VH?F)f-^WG42GF_VkQNGzl4bu z77`K;tYpH!PH%-hPt`cMZo#+ry~gl=7E&twRZT^iu&nG;^%sphU^3kh_hk!4zDM!)I~-Roz+JU7<=K%p-q*Mj?ne_hd_hsG z&Omc>!l0dm#q}nxvK=HS{%PjQn@{FLmjmr32^z|RmS zx?JOS=x}1JEo-^TId0Iy6h%pIaa8Ui1!BvzI^!ZCF;vUfP)T5MW4Z4fgw<}cNqBI9 z{yX8On-mJcK_)VjG5hU*@RurrQat3ySdNp)m`UB=oWENt@xH{QARnFLd}#^^3zGyQ zsTg;xikVDsGlk3(I#MZiK})?v;6Q+wV16p%tuIl$L5O&v+l>>kV6S9>1=+a#J6$d> zn86Y_(lGtqs~CCz8_5J4A7k~}({RkBHyNI|ASJTmEz88zcJ7j3__Uk=pJb9EaipHB z8=0h7?A@&?I~2bsn<3;O9`W_2S!95|M3M}rJQ$|4ne$N(hB0+=74edYSa(HJUO3~$ z2#=VuLrs^Q2(ej+t`BlMA_Y1gD!jX&WnpCfuM!;1BITlWRC`E)uW!rTP@Ij1J+U%9 zfxYdi%s;CRiqiuOL)c@)r)(VVzT#QL+jDObT1O)?bRRLsgVBe{H!tO?(uXx@Tc)Wb z@YETtaLb43)|Sm80`Dxw8Z61hFqhVIUZ0C&<}24!I^1eD5_--!ctTzITQweFJ=o|+ zRNgPc?I*@uaGS8Ly?$(KK?-!QQv!6uIB+e-{)KWV@xajA)8*Mmaa}IO4(MBqLwaaA z6V?EZS^6qSKB#1XS;BkjNC3SjDkv#nI}3TZbKfQ-xbny=Lepl5YFEIB+ z;PS=qD6zuIVI_gpep(uNGov{a2~5L&fBD5zbT3ngN!~ zgF?*g+%OzVQxjoHkQ50!N0@kSRi|&hAbnws;#7EOQwh6atVM`3$0_!+gg>U_aj!xHBUgDM}-3CizgX?(pn z6xJS+;qzj=s@A;B?PDLR3GC}t!)W*K=KSZgstw)<6M0UTu1j|;q|+&v6`fA^KYwyH Aod5s; diff --git a/registry/index.json b/registry/index.json index 193e6e0..e5adcd7 100644 --- a/registry/index.json +++ b/registry/index.json @@ -1,7 +1,7 @@ { "schemaVersion": 1, "name": "Scion", - "version": "0.2.0", + "version": "0.3.0", "description": "Backend pattern registry", "patterns": [ { @@ -207,6 +207,57 @@ "include": ["*.go"], "standaloneInclude": ["go.mod", "go.sum"], "status": "ready" + }, + { + "id": "migrations", + "name": "Database Migrations", + "description": "Standard-library SQL migration runner with checksums, transactions, and status handler", + "path": "registry/migrations", + "languages": ["go"], + "frameworks": [], + "tags": ["database", "migrations", "sql", "schema", "transactions"], + "version": "0.1.0", + "package": "migrations", + "source": "registry/migrations/src/go", + "defaultTarget": "internal/migrations", + "stdlibOnly": true, + "include": ["*.go"], + "standaloneInclude": ["go.mod", "go.sum"], + "status": "ready" + }, + { + "id": "metrics", + "name": "Prometheus Metrics", + "description": "Prometheus HTTP metrics with isolated registry, route cardinality limits, and /metrics handler", + "path": "registry/metrics", + "languages": ["go"], + "frameworks": ["net/http", "gin", "echo"], + "tags": ["metrics", "prometheus", "monitoring", "observability", "middleware"], + "version": "0.1.0", + "package": "metrics", + "source": "registry/metrics/src/go", + "defaultTarget": "internal/metrics", + "stdlibOnly": false, + "include": ["*.go"], + "standaloneInclude": ["go.mod", "go.sum"], + "status": "external-deps" + }, + { + "id": "problem", + "name": "Problem Details", + "description": "RFC 9457-style API error responses with safe sanitization and panic recovery", + "path": "registry/problem", + "languages": ["go"], + "frameworks": ["net/http", "gin", "echo"], + "tags": ["api", "errors", "problem-details", "rfc9457", "security"], + "version": "0.1.0", + "package": "problem", + "source": "registry/problem/src/go", + "defaultTarget": "internal/problem", + "stdlibOnly": true, + "include": ["*.go"], + "standaloneInclude": ["go.mod", "go.sum"], + "status": "ready" } ] } diff --git a/registry/metrics/README.md b/registry/metrics/README.md new file mode 100644 index 0000000..8ea5432 --- /dev/null +++ b/registry/metrics/README.md @@ -0,0 +1,54 @@ +# Metrics Module + +Prometheus HTTP metrics for `net/http` services. + +## What's Included + +- Isolated Prometheus registry +- HTTP request counter +- Request duration histogram +- In-flight request gauge +- `/metrics` handler +- Route label cardinality limits +- CRLF, null byte, raw URL, and long-label protection + +## Quick Copy + +This module uses Prometheus and should be copied in standalone mode: + +```bash +scion add metrics --standalone --to internal/metrics +``` + +## Usage + +```go +m, err := metrics.New() +if err != nil { + return err +} +_ = m.RegisterDefaults() + +http.Handle("/metrics", m.Handler()) +http.Handle("/users/", m.Middleware("/users/{id}")(usersHandler)) +``` + +Always pass a route template such as `/users/{id}`. Do not pass `r.URL.Path` +or a full URL because that can create unbounded label cardinality. + +## File Reference + +| File | Purpose | +|------|---------| +| `config.go` | Options, defaults, env loading | +| `metrics.go` | Registry and collectors | +| `middleware.go` | HTTP instrumentation middleware | +| `handler.go` | Prometheus scrape handler | +| `pentest_test.go` | Attack-scenario tests | + +## Tests + +```bash +cd registry/metrics/src/go +go test -v ./... +``` diff --git a/registry/metrics/__llms__.md b/registry/metrics/__llms__.md new file mode 100644 index 0000000..a340c16 --- /dev/null +++ b/registry/metrics/__llms__.md @@ -0,0 +1,3 @@ +# metrics module + +Prometheus-backed Go HTTP metrics module. Copy standalone with `go.mod`/`go.sum` because it depends on `github.com/prometheus/client_golang`. Use `metrics.New()`, optional `RegisterDefaults()`, `m.Handler()` for `/metrics`, and `m.Middleware("/route/{id}")`. Never pass raw URL paths as labels. Enforces route/method length checks, CRLF/null-byte rejection, and max route-cardinality overflow. diff --git a/registry/metrics/src/go/config.go b/registry/metrics/src/go/config.go new file mode 100644 index 0000000..b681fb9 --- /dev/null +++ b/registry/metrics/src/go/config.go @@ -0,0 +1,112 @@ +// Package metrics provides Prometheus HTTP metrics for copy-paste Go services. +package metrics + +import ( + "os" + "strconv" + "strings" + + "github.com/prometheus/client_golang/prometheus" +) + +// Options controls metric names, labels, and cardinality limits. +type Options struct { + Namespace string + Subsystem string + MaxRouteLen int + MaxMethodLen int + MaxRoutes int + Buckets []float64 +} + +// Defaults returns safe HTTP metric defaults. +func Defaults() Options { + return Options{ + Namespace: "scion", + Subsystem: "http", + MaxRouteLen: 128, + MaxMethodLen: 16, + MaxRoutes: 128, + Buckets: prometheus.DefBuckets, + } +} + +// FromEnv reads options from environment variables. +// +// Supported variables: +// - METRICS_NAMESPACE +// - METRICS_SUBSYSTEM +// - METRICS_MAX_ROUTE_LEN +// - METRICS_MAX_METHOD_LEN +// - METRICS_MAX_ROUTES +func FromEnv() Options { + o := Defaults() + if v := os.Getenv("METRICS_NAMESPACE"); v != "" { + o.Namespace = v + } + if v := os.Getenv("METRICS_SUBSYSTEM"); v != "" { + o.Subsystem = v + } + if v := os.Getenv("METRICS_MAX_ROUTE_LEN"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + o.MaxRouteLen = n + } + } + if v := os.Getenv("METRICS_MAX_METHOD_LEN"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + o.MaxMethodLen = n + } + } + if v := os.Getenv("METRICS_MAX_ROUTES"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + o.MaxRoutes = n + } + } + return o +} + +func (o Options) normalize() Options { + d := Defaults() + o.Namespace = safeMetricPart(o.Namespace, d.Namespace) + o.Subsystem = safeMetricPart(o.Subsystem, d.Subsystem) + if o.MaxRouteLen <= 0 { + o.MaxRouteLen = d.MaxRouteLen + } + if o.MaxRouteLen > 512 { + o.MaxRouteLen = 512 + } + if o.MaxMethodLen <= 0 { + o.MaxMethodLen = d.MaxMethodLen + } + if o.MaxMethodLen > 32 { + o.MaxMethodLen = 32 + } + if o.MaxRoutes <= 0 { + o.MaxRoutes = d.MaxRoutes + } + if o.MaxRoutes > 10000 { + o.MaxRoutes = 10000 + } + if len(o.Buckets) == 0 { + o.Buckets = d.Buckets + } + return o +} + +func safeMetricPart(value, fallback string) string { + if containsUnsafe(value) { + return fallback + } + value = strings.TrimSpace(value) + if value == "" { + return fallback + } + for i := 0; i < len(value); i++ { + c := value[i] + if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' { + continue + } + return fallback + } + return value +} diff --git a/registry/metrics/src/go/config_test.go b/registry/metrics/src/go/config_test.go new file mode 100644 index 0000000..f7b462b --- /dev/null +++ b/registry/metrics/src/go/config_test.go @@ -0,0 +1,29 @@ +package metrics + +import "testing" + +func TestOptionsNormalize(t *testing.T) { + opts := (Options{Namespace: "bad-name", Subsystem: "http"}).normalize() + if opts.Namespace != "scion" || opts.Subsystem != "http" { + t.Fatalf("unexpected metric parts: %+v", opts) + } + if opts.MaxRouteLen != 128 || opts.MaxRoutes != 128 || len(opts.Buckets) == 0 { + t.Fatalf("unexpected defaults: %+v", opts) + } +} + +func TestFromEnv(t *testing.T) { + t.Setenv("METRICS_NAMESPACE", "app") + t.Setenv("METRICS_SUBSYSTEM", "api") + t.Setenv("METRICS_MAX_ROUTE_LEN", "64") + t.Setenv("METRICS_MAX_METHOD_LEN", "8") + t.Setenv("METRICS_MAX_ROUTES", "4") + + opts := FromEnv().normalize() + if opts.Namespace != "app" || opts.Subsystem != "api" { + t.Fatalf("unexpected names: %+v", opts) + } + if opts.MaxRouteLen != 64 || opts.MaxMethodLen != 8 || opts.MaxRoutes != 4 { + t.Fatalf("unexpected limits: %+v", opts) + } +} diff --git a/registry/metrics/src/go/go.mod b/registry/metrics/src/go/go.mod new file mode 100644 index 0000000..a93d1d2 --- /dev/null +++ b/registry/metrics/src/go/go.mod @@ -0,0 +1,17 @@ +module metrics + +go 1.22 + +require github.com/prometheus/client_golang v1.20.5 + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + golang.org/x/sys v0.22.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect +) diff --git a/registry/metrics/src/go/go.sum b/registry/metrics/src/go/go.sum new file mode 100644 index 0000000..d5318cf --- /dev/null +++ b/registry/metrics/src/go/go.sum @@ -0,0 +1,24 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/registry/metrics/src/go/handler.go b/registry/metrics/src/go/handler.go new file mode 100644 index 0000000..10ed4d6 --- /dev/null +++ b/registry/metrics/src/go/handler.go @@ -0,0 +1,16 @@ +package metrics + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// Handler returns an HTTP handler for the module's isolated Prometheus +// registry. +func (m *Metrics) Handler() http.Handler { + if m == nil { + return http.NotFoundHandler() + } + return promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{}) +} diff --git a/registry/metrics/src/go/handler_test.go b/registry/metrics/src/go/handler_test.go new file mode 100644 index 0000000..b5e4d52 --- /dev/null +++ b/registry/metrics/src/go/handler_test.go @@ -0,0 +1,33 @@ +package metrics + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestHandlerExposesMetrics(t *testing.T) { + m, err := New() + if err != nil { + t.Fatalf("new: %v", err) + } + m.observe("GET", "/health", http.StatusOK, 0.01) + rec := httptest.NewRecorder() + m.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d", rec.Code) + } + if !strings.Contains(rec.Body.String(), "scion_http_requests_total") { + t.Fatalf("unexpected body: %s", rec.Body.String()) + } +} + +func TestNilHandlerIsNotFound(t *testing.T) { + var m *Metrics + rec := httptest.NewRecorder() + m.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d", rec.Code) + } +} diff --git a/registry/metrics/src/go/metrics.go b/registry/metrics/src/go/metrics.go new file mode 100644 index 0000000..6a6569e --- /dev/null +++ b/registry/metrics/src/go/metrics.go @@ -0,0 +1,157 @@ +package metrics + +import ( + "errors" + "strconv" + "strings" + "sync" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" +) + +const ( + unknownRoute = "unknown" + invalidRoute = "invalid" + overflowRoute = "overflow" +) + +// Metrics owns an isolated Prometheus registry and HTTP metric collectors. +type Metrics struct { + opts Options + + registry *prometheus.Registry + requests *prometheus.CounterVec + duration *prometheus.HistogramVec + inFlight prometheus.Gauge + + mu sync.Mutex + routes map[string]struct{} + defaultsRegistered bool +} + +// New creates an isolated metrics registry. It does not use the global +// Prometheus registry, so copied modules do not conflict with application code. +func New(opts ...Options) (*Metrics, error) { + opt := Defaults() + if len(opts) > 0 { + opt = opts[0] + } + opt = opt.normalize() + reg := prometheus.NewRegistry() + m := &Metrics{ + opts: opt, + registry: reg, + routes: make(map[string]struct{}), + requests: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: opt.Namespace, + Subsystem: opt.Subsystem, + Name: "requests_total", + Help: "Total HTTP requests by method, route, and status code.", + }, []string{"method", "route", "status"}), + duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: opt.Namespace, + Subsystem: opt.Subsystem, + Name: "request_duration_seconds", + Help: "HTTP request duration in seconds by method, route, and status code.", + Buckets: opt.Buckets, + }, []string{"method", "route", "status"}), + inFlight: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: opt.Namespace, + Subsystem: opt.Subsystem, + Name: "requests_in_flight", + Help: "Current in-flight HTTP requests.", + }), + } + if err := reg.Register(m.requests); err != nil { + return nil, err + } + if err := reg.Register(m.duration); err != nil { + return nil, err + } + if err := reg.Register(m.inFlight); err != nil { + return nil, err + } + return m, nil +} + +// RegisterDefaults registers Go runtime and process collectors once. +func (m *Metrics) RegisterDefaults() error { + if m == nil { + return errors.New("metrics: Metrics is nil") + } + m.mu.Lock() + defer m.mu.Unlock() + if m.defaultsRegistered { + return nil + } + if err := m.registry.Register(collectors.NewGoCollector()); err != nil { + return err + } + if err := m.registry.Register(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})); err != nil { + return err + } + m.defaultsRegistered = true + return nil +} + +// Registry returns the isolated Prometheus registry. +func (m *Metrics) Registry() *prometheus.Registry { + if m == nil { + return prometheus.NewRegistry() + } + return m.registry +} + +func (m *Metrics) observe(method, route string, status int, seconds float64) { + method = safeMethod(method, m.opts) + route = m.safeRoute(route) + statusLabel := strconv.Itoa(status) + m.requests.WithLabelValues(method, route, statusLabel).Inc() + m.duration.WithLabelValues(method, route, statusLabel).Observe(seconds) +} + +func (m *Metrics) safeRoute(route string) string { + if containsUnsafe(route) { + route = invalidRoute + } + route = strings.TrimSpace(route) + switch { + case route == "": + route = unknownRoute + case len(route) > m.opts.MaxRouteLen, strings.Contains(route, "?"), strings.Contains(route, "://"): + route = invalidRoute + } + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.routes[route]; ok { + return route + } + if len(m.routes) >= m.opts.MaxRoutes { + return overflowRoute + } + m.routes[route] = struct{}{} + return route +} + +func safeMethod(method string, opts Options) string { + if containsUnsafe(method) { + return "UNKNOWN" + } + method = strings.TrimSpace(method) + if method == "" || len(method) > opts.MaxMethodLen { + return "UNKNOWN" + } + for i := 0; i < len(method); i++ { + c := method[i] + if c >= 'A' && c <= 'Z' { + continue + } + return "UNKNOWN" + } + return method +} + +func containsUnsafe(value string) bool { + return strings.ContainsAny(value, "\r\n\x00") +} diff --git a/registry/metrics/src/go/metrics_test.go b/registry/metrics/src/go/metrics_test.go new file mode 100644 index 0000000..f0e4ff5 --- /dev/null +++ b/registry/metrics/src/go/metrics_test.go @@ -0,0 +1,63 @@ +package metrics + +import ( + "strings" + "testing" +) + +func TestNewRegistersCollectors(t *testing.T) { + m, err := New(Options{Namespace: "app", Subsystem: "api"}) + if err != nil { + t.Fatalf("new: %v", err) + } + m.observe("GET", "/users/{id}", 200, 0.1) + families, err := m.Registry().Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + found := false + for _, family := range families { + if family.GetName() == "app_api_requests_total" { + found = true + } + } + if !found { + t.Fatalf("requests metric not found") + } +} + +func TestRegisterDefaultsIsIdempotent(t *testing.T) { + m, err := New() + if err != nil { + t.Fatalf("new: %v", err) + } + if err := m.RegisterDefaults(); err != nil { + t.Fatalf("first register: %v", err) + } + if err := m.RegisterDefaults(); err != nil { + t.Fatalf("second register: %v", err) + } +} + +func TestRouteCardinalityOverflow(t *testing.T) { + m, err := New(Options{MaxRoutes: 1}) + if err != nil { + t.Fatalf("new: %v", err) + } + if got := m.safeRoute("/first"); got != "/first" { + t.Fatalf("first route = %q", got) + } + if got := m.safeRoute("/second"); got != overflowRoute { + t.Fatalf("second route = %q", got) + } +} + +func TestSafeMethodRejectsUnexpectedChars(t *testing.T) { + got := safeMethod("GET\r\n", Defaults()) + if got != "UNKNOWN" { + t.Fatalf("method = %q", got) + } + if strings.ContainsAny(got, "\r\n\x00") { + t.Fatalf("unsafe method = %q", got) + } +} diff --git a/registry/metrics/src/go/middleware.go b/registry/metrics/src/go/middleware.go new file mode 100644 index 0000000..652f811 --- /dev/null +++ b/registry/metrics/src/go/middleware.go @@ -0,0 +1,66 @@ +package metrics + +import ( + "net/http" + "time" +) + +// Middleware records request count, latency, in-flight count, method, route, +// and status. Pass a route template such as "/users/{id}", not r.URL.Path. +func (m *Metrics) Middleware(route string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return m.Instrument(route, next) + } +} + +// Instrument wraps a handler and records Prometheus metrics for it. +func (m *Metrics) Instrument(route string, next http.Handler) http.Handler { + if m == nil { + return next + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + m.inFlight.Inc() + defer m.inFlight.Dec() + + lw := &responseWriter{ResponseWriter: w} + defer func() { + if recovered := recover(); recovered != nil { + if lw.status == 0 { + lw.status = http.StatusInternalServerError + } + m.observe(r.Method, route, lw.status, time.Since(start).Seconds()) + panic(recovered) + } + status := lw.status + if status == 0 { + status = http.StatusOK + } + m.observe(r.Method, route, status, time.Since(start).Seconds()) + }() + next.ServeHTTP(lw, r) + }) +} + +type responseWriter struct { + http.ResponseWriter + status int +} + +func (w *responseWriter) WriteHeader(status int) { + if w.status == 0 { + w.status = status + w.ResponseWriter.WriteHeader(status) + } +} + +func (w *responseWriter) Write(b []byte) (int, error) { + if w.status == 0 { + w.WriteHeader(http.StatusOK) + } + return w.ResponseWriter.Write(b) +} + +func (w *responseWriter) Unwrap() http.ResponseWriter { + return w.ResponseWriter +} diff --git a/registry/metrics/src/go/middleware_test.go b/registry/metrics/src/go/middleware_test.go new file mode 100644 index 0000000..40d80d9 --- /dev/null +++ b/registry/metrics/src/go/middleware_test.go @@ -0,0 +1,78 @@ +package metrics + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestMiddlewareRecordsStatus(t *testing.T) { + m, err := New() + if err != nil { + t.Fatalf("new: %v", err) + } + h := m.Middleware("/users/{id}")(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusCreated) + })) + h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/users/123", nil)) + + rec := httptest.NewRecorder() + m.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + body := rec.Body.String() + if !strings.Contains(body, `scion_http_requests_total{method="POST",route="/users/{id}",status="201"} 1`) { + t.Fatalf("metric missing from body:\n%s", body) + } +} + +func TestMiddlewareDefaultsImplicitStatus(t *testing.T) { + m, err := New() + if err != nil { + t.Fatalf("new: %v", err) + } + h := m.Instrument("/ok", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("ok")) + })) + h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/ok", nil)) + rec := httptest.NewRecorder() + m.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + if !strings.Contains(rec.Body.String(), `status="200"`) { + t.Fatalf("expected implicit 200 metric:\n%s", rec.Body.String()) + } +} + +func TestMiddlewareRecordsPanicBeforeReraising(t *testing.T) { + m, err := New() + if err != nil { + t.Fatalf("new: %v", err) + } + h := m.Instrument("/panic", http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + panic("boom") + })) + func() { + defer func() { + if recover() == nil { + t.Fatalf("expected panic") + } + }() + h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/panic", nil)) + }() + rec := httptest.NewRecorder() + m.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + if !strings.Contains(rec.Body.String(), `route="/panic",status="500"`) { + t.Fatalf("panic metric missing:\n%s", rec.Body.String()) + } +} + +func TestResponseWriterSuppressesDuplicateHeaderAndUnwraps(t *testing.T) { + rec := httptest.NewRecorder() + lw := &responseWriter{ResponseWriter: rec} + lw.WriteHeader(http.StatusCreated) + lw.WriteHeader(http.StatusInternalServerError) + if rec.Code != http.StatusCreated || lw.status != http.StatusCreated { + t.Fatalf("status rec=%d wrapped=%d", rec.Code, lw.status) + } + if lw.Unwrap() != rec { + t.Fatalf("unwrap did not return underlying writer") + } +} diff --git a/registry/metrics/src/go/pentest_test.go b/registry/metrics/src/go/pentest_test.go new file mode 100644 index 0000000..940a32d --- /dev/null +++ b/registry/metrics/src/go/pentest_test.go @@ -0,0 +1,56 @@ +package metrics + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestPentestRejectsRawURLRouteLabels(t *testing.T) { + m, err := New() + if err != nil { + t.Fatalf("new: %v", err) + } + h := m.Middleware("/users?id=secret")(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/users?id=secret", nil)) + rec := httptest.NewRecorder() + m.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + body := rec.Body.String() + if strings.Contains(body, "secret") || !strings.Contains(body, `route="invalid"`) { + t.Fatalf("unsafe route label handling:\n%s", body) + } +} + +func TestPentestRejectsCRLFAndNullLabels(t *testing.T) { + m, err := New() + if err != nil { + t.Fatalf("new: %v", err) + } + m.observe("GET\r\n", "/evil\x00route", http.StatusOK, 0.01) + rec := httptest.NewRecorder() + m.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + body := rec.Body.String() + if strings.ContainsAny(body, "\r\x00") { + t.Fatalf("unsafe bytes in metrics: %q", body) + } + if !strings.Contains(body, `method="UNKNOWN"`) || !strings.Contains(body, `route="invalid"`) { + t.Fatalf("unsafe labels not normalized:\n%s", body) + } +} + +func TestPentestLimitsRouteCardinality(t *testing.T) { + m, err := New(Options{MaxRoutes: 1}) + if err != nil { + t.Fatalf("new: %v", err) + } + m.observe("GET", "/a", 200, 0.01) + m.observe("GET", "/b", 200, 0.01) + rec := httptest.NewRecorder() + m.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + if !strings.Contains(rec.Body.String(), `route="overflow"`) { + t.Fatalf("overflow route missing:\n%s", rec.Body.String()) + } +} diff --git a/registry/migrations/README.md b/registry/migrations/README.md new file mode 100644 index 0000000..6de8635 --- /dev/null +++ b/registry/migrations/README.md @@ -0,0 +1,67 @@ +# Migrations Module + +Standard-library-only SQL migration runner for `database/sql`. + +## What's Included + +- `fs.FS` migration discovery +- Paired `YYYYMMDDHHMMSS_name.up.sql` / `.down.sql` files +- Schema history table with version, name, checksum, and timestamp +- Transactional `Up`, `Down`, and `Status` +- Safe table-name and filename validation +- Status HTTP handler with panic recovery + +## Quick Copy + +```bash +cp -r registry/migrations/src/go/*.go yourproject/internal/migrations/ +``` + +Or with the Scion CLI: + +```bash +scion add migrations --to internal/migrations +``` + +## Usage + +```go +//go:embed migrations/*.sql +var migrationFS embed.FS + +m, err := migrations.New(migrationFS, migrations.Options{ + Dir: "migrations", +}) +if err != nil { + return err +} +if err := m.Up(ctx, db); err != nil { + return err +} +``` + +For PostgreSQL-style placeholders: + +```go +m, _ := migrations.New(migrationFS, migrations.Options{ + Dir: "migrations", + Placeholder: migrations.DollarPlaceholder, +}) +``` + +## File Reference + +| File | Purpose | +|------|---------| +| `config.go` | Options, defaults, env loading, placeholder helpers | +| `file.go` | Migration filename parsing, SQL loading, checksums | +| `migrator.go` | `Up`, `Down`, `Status`, and schema history bookkeeping | +| `handler.go` | HTTP status handler | +| `pentest_test.go` | Attack-scenario tests | + +## Tests + +```bash +cd registry/migrations/src/go +go test -v ./... +``` diff --git a/registry/migrations/__llms__.md b/registry/migrations/__llms__.md new file mode 100644 index 0000000..5a239ad --- /dev/null +++ b/registry/migrations/__llms__.md @@ -0,0 +1,3 @@ +# migrations module + +Zero-dependency Go SQL migration runner. Copy `src/go/*.go` into `internal/migrations`. Use `New(fsys, Options{Dir: "migrations"})`, then call `Up(ctx, db)`, `Down(ctx, db, steps)`, or `Status(ctx, db)`. Migration files must be named `YYYYMMDDHHMMSS_name.up.sql` with optional paired `.down.sql`. Validates filenames, table names, size limits, null bytes, and path traversal. Use `DollarPlaceholder` for PostgreSQL. diff --git a/registry/migrations/src/go/config.go b/registry/migrations/src/go/config.go new file mode 100644 index 0000000..4c7ccf2 --- /dev/null +++ b/registry/migrations/src/go/config.go @@ -0,0 +1,135 @@ +// Package migrations provides a small, standard-library-only database migration +// runner for copy-paste Go services. +package migrations + +import ( + "os" + "path" + "strconv" + "strings" + "time" +) + +// Placeholder returns a SQL placeholder for the 1-based argument position. +type Placeholder func(position int) string + +// Options configures migration discovery and SQL bookkeeping. +type Options struct { + // Dir is the directory inside the provided fs.FS that contains migration + // files. Defaults to ".". + Dir string + // TableName is the trusted schema history table name. Dot-qualified names + // such as "public.schema_migrations" are allowed. + TableName string + // MaxMigrations caps the number of migration files loaded from Dir. + MaxMigrations int + // MaxSQLBytes caps the size of each migration SQL file. + MaxSQLBytes int64 + // DisableTransactions executes migration SQL and bookkeeping outside an + // explicit database transaction. Keep false unless your driver or migration + // statement cannot run in a transaction. + DisableTransactions bool + // Placeholder controls bind marker syntax. Use DollarPlaceholder for + // PostgreSQL-style drivers. + Placeholder Placeholder + // Now supplies timestamps for schema history rows. Defaults to time.Now. + Now func() time.Time +} + +// Defaults returns secure defaults for local SQL migration files. +func Defaults() Options { + return Options{ + Dir: ".", + TableName: "schema_migrations", + MaxMigrations: 512, + MaxSQLBytes: 1 << 20, + Placeholder: QuestionPlaceholder, + Now: time.Now, + } +} + +// FromEnv reads migration options from environment variables. +// +// Supported variables: +// - MIGRATIONS_DIR +// - MIGRATIONS_TABLE +// - MIGRATIONS_MAX_FILES +// - MIGRATIONS_MAX_SQL_BYTES +// - MIGRATIONS_DISABLE_TRANSACTIONS +// - MIGRATIONS_PLACEHOLDER_STYLE ("question" or "dollar") +func FromEnv() Options { + o := Defaults() + if v := os.Getenv("MIGRATIONS_DIR"); v != "" { + o.Dir = v + } + if v := os.Getenv("MIGRATIONS_TABLE"); v != "" { + o.TableName = v + } + if v := os.Getenv("MIGRATIONS_MAX_FILES"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + o.MaxMigrations = n + } + } + if v := os.Getenv("MIGRATIONS_MAX_SQL_BYTES"); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil { + o.MaxSQLBytes = n + } + } + if v := os.Getenv("MIGRATIONS_DISABLE_TRANSACTIONS"); v != "" { + o.DisableTransactions = strings.EqualFold(v, "true") + } + if v := strings.ToLower(os.Getenv("MIGRATIONS_PLACEHOLDER_STYLE")); v == "dollar" { + o.Placeholder = DollarPlaceholder + } + return o +} + +func (o Options) normalize() (Options, error) { + d := Defaults() + if o.Dir == "" { + o.Dir = d.Dir + } + if o.TableName == "" { + o.TableName = d.TableName + } + if o.MaxMigrations <= 0 { + o.MaxMigrations = d.MaxMigrations + } + if o.MaxMigrations > 10000 { + o.MaxMigrations = 10000 + } + if o.MaxSQLBytes <= 0 { + o.MaxSQLBytes = d.MaxSQLBytes + } + if o.MaxSQLBytes > 64<<20 { + o.MaxSQLBytes = 64 << 20 + } + if o.Placeholder == nil { + o.Placeholder = d.Placeholder + } + if o.Now == nil { + o.Now = d.Now + } + if err := validateDir(o.Dir); err != nil { + return Options{}, err + } + o.Dir = path.Clean(o.Dir) + if err := validateTableName(o.TableName); err != nil { + return Options{}, err + } + return o, nil +} + +// QuestionPlaceholder returns "?" for MySQL, SQLite, and other drivers that +// use anonymous placeholders. +func QuestionPlaceholder(_ int) string { + return "?" +} + +// DollarPlaceholder returns PostgreSQL-style placeholders: $1, $2, ... +func DollarPlaceholder(position int) string { + if position < 1 { + position = 1 + } + return "$" + strconv.Itoa(position) +} diff --git a/registry/migrations/src/go/config_test.go b/registry/migrations/src/go/config_test.go new file mode 100644 index 0000000..9b0c955 --- /dev/null +++ b/registry/migrations/src/go/config_test.go @@ -0,0 +1,62 @@ +package migrations + +import ( + "testing" + "time" +) + +func TestDefaultsNormalize(t *testing.T) { + opts, err := (Options{}).normalize() + if err != nil { + t.Fatalf("normalize: %v", err) + } + if opts.Dir != "." || opts.TableName != "schema_migrations" { + t.Fatalf("unexpected defaults: %+v", opts) + } + if opts.Placeholder(2) != "?" { + t.Fatalf("unexpected placeholder") + } + if opts.Now == nil || opts.Now().IsZero() { + t.Fatalf("missing Now") + } +} + +func TestFromEnv(t *testing.T) { + t.Setenv("MIGRATIONS_DIR", "db/migrations") + t.Setenv("MIGRATIONS_TABLE", "public.schema_migrations") + t.Setenv("MIGRATIONS_MAX_FILES", "10") + t.Setenv("MIGRATIONS_MAX_SQL_BYTES", "2048") + t.Setenv("MIGRATIONS_DISABLE_TRANSACTIONS", "true") + t.Setenv("MIGRATIONS_PLACEHOLDER_STYLE", "dollar") + + opts, err := FromEnv().normalize() + if err != nil { + t.Fatalf("normalize: %v", err) + } + if opts.Dir != "db/migrations" || opts.TableName != "public.schema_migrations" { + t.Fatalf("unexpected env options: %+v", opts) + } + if opts.MaxMigrations != 10 || opts.MaxSQLBytes != 2048 || !opts.DisableTransactions { + t.Fatalf("unexpected limits: %+v", opts) + } + if got := opts.Placeholder(3); got != "$3" { + t.Fatalf("placeholder = %q", got) + } +} + +func TestNormalizeRejectsUnsafeTable(t *testing.T) { + _, err := Options{TableName: "schema_migrations;DROP", Now: time.Now}.normalize() + if err == nil { + t.Fatalf("expected unsafe table error") + } +} + +func TestNormalizeCleansSafeDirectory(t *testing.T) { + opts, err := Options{Dir: "db/./migrations"}.normalize() + if err != nil { + t.Fatalf("normalize: %v", err) + } + if opts.Dir != "db/migrations" { + t.Fatalf("dir = %q", opts.Dir) + } +} diff --git a/registry/migrations/src/go/file.go b/registry/migrations/src/go/file.go new file mode 100644 index 0000000..b348913 --- /dev/null +++ b/registry/migrations/src/go/file.go @@ -0,0 +1,253 @@ +package migrations + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "path" + "sort" + "strconv" + "strings" +) + +const ( + maxMigrationNameLen = 128 + versionDigits = 14 +) + +// Migration is a paired up/down SQL migration loaded from fs.FS. +type Migration struct { + Version int64 + Name string + UpSQL string + DownSQL string + Checksum string +} + +// Load reads and validates migrations from fsys using Defaults(). +func Load(fsys fs.FS, opts ...Options) ([]Migration, error) { + opt := Defaults() + if len(opts) > 0 { + opt = opts[0] + } + opt, err := opt.normalize() + if err != nil { + return nil, err + } + return loadMigrations(fsys, opt) +} + +func loadMigrations(fsys fs.FS, opt Options) ([]Migration, error) { + if fsys == nil { + return nil, errors.New("migrations: fsys is nil") + } + entries, err := fs.ReadDir(fsys, opt.Dir) + if err != nil { + return nil, fmt.Errorf("migrations: read dir: %w", err) + } + if len(entries) > opt.MaxMigrations*2 { + return nil, errors.New("migrations: too many files in migration directory") + } + + byVersion := make(map[int64]*Migration) + seenFiles := 0 + for _, entry := range entries { + if entry.IsDir() { + continue + } + base := entry.Name() + direction := "" + switch { + case strings.HasSuffix(base, ".up.sql"): + direction = "up" + case strings.HasSuffix(base, ".down.sql"): + direction = "down" + default: + continue + } + seenFiles++ + if seenFiles > opt.MaxMigrations*2 { + return nil, errors.New("migrations: too many migration files") + } + version, name, err := parseMigrationName(base, direction) + if err != nil { + return nil, err + } + sqlText, err := readMigrationSQL(fsys, opt, base) + if err != nil { + return nil, err + } + m := byVersion[version] + if m == nil { + m = &Migration{Version: version, Name: name} + byVersion[version] = m + } + if m.Name != name { + return nil, fmt.Errorf("migrations: version %d has conflicting names", version) + } + if direction == "up" { + if m.UpSQL != "" { + return nil, fmt.Errorf("migrations: duplicate up migration for version %d", version) + } + m.UpSQL = sqlText + m.Checksum = checksumSQL(sqlText) + } else { + if m.DownSQL != "" { + return nil, fmt.Errorf("migrations: duplicate down migration for version %d", version) + } + m.DownSQL = sqlText + } + } + + migrations := make([]Migration, 0, len(byVersion)) + for _, m := range byVersion { + if m.UpSQL == "" { + return nil, fmt.Errorf("migrations: version %d is missing an up migration", m.Version) + } + migrations = append(migrations, *m) + } + sort.Slice(migrations, func(i, j int) bool { + return migrations[i].Version < migrations[j].Version + }) + return migrations, nil +} + +func parseMigrationName(base, direction string) (int64, string, error) { + if containsUnsafe(base) { + return 0, "", errors.New("migrations: file name contains unsafe characters") + } + suffix := "." + direction + ".sql" + stem := strings.TrimSuffix(base, suffix) + if len(stem) <= versionDigits+1 || stem[versionDigits] != '_' { + return 0, "", fmt.Errorf("migrations: invalid file name %q", base) + } + rawVersion := stem[:versionDigits] + for i := 0; i < len(rawVersion); i++ { + if rawVersion[i] < '0' || rawVersion[i] > '9' { + return 0, "", fmt.Errorf("migrations: invalid version in %q", base) + } + } + version, err := strconv.ParseInt(rawVersion, 10, 64) + if err != nil { + return 0, "", fmt.Errorf("migrations: invalid version in %q: %w", base, err) + } + name := stem[versionDigits+1:] + if err := validateMigrationName(name); err != nil { + return 0, "", fmt.Errorf("migrations: invalid file name %q: %w", base, err) + } + return version, name, nil +} + +func readMigrationSQL(fsys fs.FS, opt Options, base string) (string, error) { + full := path.Join(opt.Dir, base) + data, err := fs.ReadFile(fsys, full) + if err != nil { + return "", fmt.Errorf("migrations: read %s: %w", base, err) + } + if int64(len(data)) > opt.MaxSQLBytes { + return "", fmt.Errorf("migrations: %s exceeds max SQL size", base) + } + if strings.ContainsRune(string(data), '\x00') { + return "", fmt.Errorf("migrations: %s contains a null byte", base) + } + sqlText := strings.TrimSpace(string(data)) + if sqlText == "" { + return "", fmt.Errorf("migrations: %s is empty", base) + } + return sqlText, nil +} + +func checksumSQL(sqlText string) string { + sum := sha256.Sum256([]byte(sqlText)) + return hex.EncodeToString(sum[:]) +} + +func validateDir(dir string) error { + if containsUnsafe(dir) { + return errors.New("migrations: directory contains unsafe characters") + } + if strings.Contains(dir, "..") { + return errors.New("migrations: directory contains path traversal") + } + clean := path.Clean(dir) + if clean == ".." || strings.HasPrefix(clean, "../") || strings.HasPrefix(clean, "/") { + return errors.New("migrations: directory must stay within fs root") + } + return nil +} + +func validateMigrationName(name string) error { + if name == "" { + return errors.New("name is empty") + } + if len(name) > maxMigrationNameLen { + return errors.New("name is too long") + } + if strings.Contains(name, "..") { + return errors.New("name contains path traversal") + } + for i := 0; i < len(name); i++ { + c := name[i] + if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-' { + continue + } + return errors.New("name contains unsafe characters") + } + return nil +} + +func validateTableName(table string) error { + if table == "" { + return errors.New("migrations: table name is empty") + } + if len(table) > maxMigrationNameLen { + return errors.New("migrations: table name is too long") + } + if containsUnsafe(table) || strings.Contains(table, "..") { + return errors.New("migrations: table name contains unsafe characters") + } + parts := strings.Split(table, ".") + if len(parts) > 3 { + return errors.New("migrations: table name has too many segments") + } + for _, part := range parts { + if !isIdentifier(part) { + return errors.New("migrations: table name contains unsafe identifier") + } + } + return nil +} + +func containsUnsafe(s string) bool { + return strings.ContainsAny(s, "\r\n\x00") +} + +func isIdentifier(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c == '_': + continue + case i == 0 && isASCIILetter(c): + continue + case i > 0 && (isASCIILetter(c) || isASCIIDigit(c)): + continue + default: + return false + } + } + return true +} + +func isASCIILetter(c byte) bool { + return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') +} + +func isASCIIDigit(c byte) bool { + return '0' <= c && c <= '9' +} diff --git a/registry/migrations/src/go/file_test.go b/registry/migrations/src/go/file_test.go new file mode 100644 index 0000000..0f3fb57 --- /dev/null +++ b/registry/migrations/src/go/file_test.go @@ -0,0 +1,55 @@ +package migrations + +import ( + "testing" + "testing/fstest" +) + +func TestLoadMigrationsPairsAndSorts(t *testing.T) { + fsys := fstest.MapFS{ + "20260101000002_add_posts.up.sql": {Data: []byte("CREATE TABLE posts(id BIGINT);")}, + "20260101000002_add_posts.down.sql": {Data: []byte("DROP TABLE posts;")}, + "20260101000001_add_users.up.sql": {Data: []byte("CREATE TABLE users(id BIGINT);")}, + } + migrations, err := Load(fsys) + if err != nil { + t.Fatalf("load: %v", err) + } + if len(migrations) != 2 { + t.Fatalf("len = %d", len(migrations)) + } + if migrations[0].Name != "add_users" || migrations[1].Name != "add_posts" { + t.Fatalf("unexpected order: %+v", migrations) + } + if migrations[1].DownSQL == "" || migrations[0].Checksum == "" { + t.Fatalf("missing migration fields: %+v", migrations) + } +} + +func TestLoadRejectsMissingUp(t *testing.T) { + fsys := fstest.MapFS{ + "20260101000001_add_users.down.sql": {Data: []byte("DROP TABLE users;")}, + } + if _, err := Load(fsys); err == nil { + t.Fatalf("expected missing up error") + } +} + +func TestLoadRejectsInvalidFilename(t *testing.T) { + fsys := fstest.MapFS{ + "20260101000001_bad.name.up.sql": {Data: []byte("SELECT 1;")}, + } + if _, err := Load(fsys); err == nil { + t.Fatalf("expected invalid filename error") + } +} + +func TestLoadRejectsLargeSQL(t *testing.T) { + fsys := fstest.MapFS{ + "20260101000001_add_users.up.sql": {Data: []byte("SELECT 12345;")}, + } + _, err := Load(fsys, Options{MaxSQLBytes: 4}) + if err == nil { + t.Fatalf("expected max SQL size error") + } +} diff --git a/registry/migrations/src/go/go.mod b/registry/migrations/src/go/go.mod new file mode 100644 index 0000000..fd9450b --- /dev/null +++ b/registry/migrations/src/go/go.mod @@ -0,0 +1,3 @@ +module migrations + +go 1.22 diff --git a/registry/migrations/src/go/handler.go b/registry/migrations/src/go/handler.go new file mode 100644 index 0000000..b71e057 --- /dev/null +++ b/registry/migrations/src/go/handler.go @@ -0,0 +1,31 @@ +package migrations + +import ( + "database/sql" + "encoding/json" + "net/http" +) + +// StatusHandler returns an HTTP handler that reports migration status as JSON. +// The handler recovers from panics and never exposes request headers or bodies. +func (m *Migrator) StatusHandler(db *sql.DB) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if recover() != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal server error"}) + } + }() + statuses, err := m.Status(r.Context(), db) + if err != nil { + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "migration status unavailable"}) + return + } + writeJSON(w, http.StatusOK, statuses) + }) +} + +func writeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} diff --git a/registry/migrations/src/go/handler_test.go b/registry/migrations/src/go/handler_test.go new file mode 100644 index 0000000..184b78d --- /dev/null +++ b/registry/migrations/src/go/handler_test.go @@ -0,0 +1,45 @@ +package migrations + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "testing/fstest" +) + +func TestStatusHandlerReturnsJSON(t *testing.T) { + db, _ := openFakeDB(t) + fsys := fstest.MapFS{ + "20260101000001_add_users.up.sql": {Data: []byte("CREATE TABLE users(id BIGINT);")}, + } + m, err := New(fsys) + if err != nil { + t.Fatalf("new: %v", err) + } + rec := httptest.NewRecorder() + m.StatusHandler(db).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/migrations", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String()) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("content-type = %q", ct) + } + if !strings.Contains(rec.Body.String(), `"version":20260101000001`) { + t.Fatalf("unexpected body: %s", rec.Body.String()) + } +} + +func TestStatusHandlerRecoversPanic(t *testing.T) { + m, err := New(fstest.MapFS{ + "20260101000001_add_users.up.sql": {Data: []byte("CREATE TABLE users(id BIGINT);")}, + }) + if err != nil { + t.Fatalf("new: %v", err) + } + rec := httptest.NewRecorder() + m.StatusHandler(nil).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/migrations", nil)) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d", rec.Code) + } +} diff --git a/registry/migrations/src/go/migrator.go b/registry/migrations/src/go/migrator.go new file mode 100644 index 0000000..95ca6c1 --- /dev/null +++ b/registry/migrations/src/go/migrator.go @@ -0,0 +1,324 @@ +package migrations + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io/fs" + "sort" + "time" +) + +// Migrator applies SQL migrations and records applied versions in a schema +// history table. +type Migrator struct { + fsys fs.FS + opts Options +} + +// AppliedMigration is a row from the schema history table. +type AppliedMigration struct { + Version int64 + Name string + Checksum string + AppliedAt sql.NullTime +} + +// Status reports the local file and database state for a migration version. +type Status struct { + Version int64 `json:"version"` + Name string `json:"name"` + Applied bool `json:"applied"` + AppliedAt *time.Time `json:"applied_at,omitempty"` + Checksum string `json:"checksum,omitempty"` + ChecksumMismatch bool `json:"checksum_mismatch,omitempty"` + Missing bool `json:"missing,omitempty"` +} + +type execer interface { + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) +} + +// New returns a migrator over fsys. If opts is omitted, Defaults() is used. +func New(fsys fs.FS, opts ...Options) (*Migrator, error) { + opt := Defaults() + if len(opts) > 0 { + opt = opts[0] + } + opt, err := opt.normalize() + if err != nil { + return nil, err + } + if fsys == nil { + return nil, errors.New("migrations: fsys is nil") + } + return &Migrator{fsys: fsys, opts: opt}, nil +} + +// Up applies all unapplied migrations in ascending version order. +func (m *Migrator) Up(ctx context.Context, db *sql.DB) error { + if db == nil { + return errors.New("migrations: db is nil") + } + if err := m.ensureTable(ctx, db); err != nil { + return err + } + migrations, err := loadMigrations(m.fsys, m.opts) + if err != nil { + return err + } + applied, err := m.appliedMap(ctx, db) + if err != nil { + return err + } + for _, migration := range migrations { + row, ok := applied[migration.Version] + if ok { + if row.Checksum != migration.Checksum { + return fmt.Errorf("migrations: checksum mismatch for version %d", migration.Version) + } + continue + } + if err := m.applyUp(ctx, db, migration); err != nil { + return err + } + } + return nil +} + +// Down rolls back the latest applied migrations. Steps must be positive. +func (m *Migrator) Down(ctx context.Context, db *sql.DB, steps int) error { + if db == nil { + return errors.New("migrations: db is nil") + } + if steps < 1 { + return errors.New("migrations: down steps must be positive") + } + if err := m.ensureTable(ctx, db); err != nil { + return err + } + local, err := loadMigrations(m.fsys, m.opts) + if err != nil { + return err + } + localByVersion := make(map[int64]Migration, len(local)) + for _, migration := range local { + localByVersion[migration.Version] = migration + } + applied, err := m.appliedRows(ctx, db) + if err != nil { + return err + } + sort.Slice(applied, func(i, j int) bool { + return applied[i].Version > applied[j].Version + }) + if steps > len(applied) { + steps = len(applied) + } + for i := 0; i < steps; i++ { + row := applied[i] + migration, ok := localByVersion[row.Version] + if !ok { + return fmt.Errorf("migrations: applied version %d has no local migration file", row.Version) + } + if migration.DownSQL == "" { + return fmt.Errorf("migrations: version %d has no down migration", row.Version) + } + if err := m.applyDown(ctx, db, migration); err != nil { + return err + } + } + return nil +} + +// Status returns the merged local-file and applied-database state. +func (m *Migrator) Status(ctx context.Context, db *sql.DB) ([]Status, error) { + if db == nil { + return nil, errors.New("migrations: db is nil") + } + if err := m.ensureTable(ctx, db); err != nil { + return nil, err + } + local, err := loadMigrations(m.fsys, m.opts) + if err != nil { + return nil, err + } + applied, err := m.appliedMap(ctx, db) + if err != nil { + return nil, err + } + seen := make(map[int64]bool, len(local)) + statuses := make([]Status, 0, len(local)+len(applied)) + for _, migration := range local { + seen[migration.Version] = true + row, ok := applied[migration.Version] + statuses = append(statuses, Status{ + Version: migration.Version, + Name: migration.Name, + Applied: ok, + AppliedAt: appliedAt(row.AppliedAt), + Checksum: migration.Checksum, + ChecksumMismatch: ok && row.Checksum != migration.Checksum, + }) + } + for version, row := range applied { + if seen[version] { + continue + } + statuses = append(statuses, Status{ + Version: row.Version, + Name: row.Name, + Applied: true, + AppliedAt: appliedAt(row.AppliedAt), + Checksum: row.Checksum, + Missing: true, + }) + } + sort.Slice(statuses, func(i, j int) bool { + return statuses[i].Version < statuses[j].Version + }) + return statuses, nil +} + +func (m *Migrator) applyUp(ctx context.Context, db *sql.DB, migration Migration) error { + if m.opts.DisableTransactions { + if _, err := db.ExecContext(ctx, migration.UpSQL); err != nil { + return fmt.Errorf("migrations: apply version %d: %w", migration.Version, err) + } + return m.insertApplied(ctx, db, migration) + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("migrations: begin tx: %w", err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback() + } + }() + if _, err := tx.ExecContext(ctx, migration.UpSQL); err != nil { + return fmt.Errorf("migrations: apply version %d: %w", migration.Version, err) + } + if err := m.insertApplied(ctx, tx, migration); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("migrations: commit version %d: %w", migration.Version, err) + } + committed = true + return nil +} + +func (m *Migrator) applyDown(ctx context.Context, db *sql.DB, migration Migration) error { + if m.opts.DisableTransactions { + if _, err := db.ExecContext(ctx, migration.DownSQL); err != nil { + return fmt.Errorf("migrations: rollback version %d: %w", migration.Version, err) + } + return m.deleteApplied(ctx, db, migration.Version) + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("migrations: begin tx: %w", err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback() + } + }() + if _, err := tx.ExecContext(ctx, migration.DownSQL); err != nil { + return fmt.Errorf("migrations: rollback version %d: %w", migration.Version, err) + } + if err := m.deleteApplied(ctx, tx, migration.Version); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("migrations: commit rollback %d: %w", migration.Version, err) + } + committed = true + return nil +} + +func (m *Migrator) ensureTable(ctx context.Context, db execer) error { + query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( +version BIGINT PRIMARY KEY, +name VARCHAR(128) NOT NULL, +checksum VARCHAR(64) NOT NULL, +applied_at TIMESTAMP NOT NULL +)`, m.opts.TableName) + if _, err := db.ExecContext(ctx, query); err != nil { + return fmt.Errorf("migrations: ensure schema table: %w", err) + } + return nil +} + +func (m *Migrator) insertApplied(ctx context.Context, db execer, migration Migration) error { + query := fmt.Sprintf("INSERT INTO %s (version, name, checksum, applied_at) VALUES (%s, %s, %s, %s)", + m.opts.TableName, + m.opts.Placeholder(1), + m.opts.Placeholder(2), + m.opts.Placeholder(3), + m.opts.Placeholder(4), + ) + _, err := db.ExecContext(ctx, query, migration.Version, migration.Name, migration.Checksum, m.opts.Now().UTC()) + if err != nil { + return fmt.Errorf("migrations: record version %d: %w", migration.Version, err) + } + return nil +} + +func (m *Migrator) deleteApplied(ctx context.Context, db execer, version int64) error { + query := fmt.Sprintf("DELETE FROM %s WHERE version = %s", m.opts.TableName, m.opts.Placeholder(1)) + _, err := db.ExecContext(ctx, query, version) + if err != nil { + return fmt.Errorf("migrations: delete version %d: %w", version, err) + } + return nil +} + +func (m *Migrator) appliedMap(ctx context.Context, db *sql.DB) (map[int64]AppliedMigration, error) { + rows, err := m.appliedRows(ctx, db) + if err != nil { + return nil, err + } + out := make(map[int64]AppliedMigration, len(rows)) + for _, row := range rows { + out[row.Version] = row + } + return out, nil +} + +func (m *Migrator) appliedRows(ctx context.Context, db *sql.DB) ([]AppliedMigration, error) { + query := fmt.Sprintf("SELECT version, name, checksum, applied_at FROM %s ORDER BY version ASC", m.opts.TableName) + rows, err := db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("migrations: query applied versions: %w", err) + } + defer rows.Close() + + var out []AppliedMigration + for rows.Next() { + if len(out) >= m.opts.MaxMigrations { + return nil, errors.New("migrations: too many applied migration rows") + } + var row AppliedMigration + if err := rows.Scan(&row.Version, &row.Name, &row.Checksum, &row.AppliedAt); err != nil { + return nil, fmt.Errorf("migrations: scan applied version: %w", err) + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("migrations: read applied versions: %w", err) + } + return out, nil +} + +func appliedAt(value sql.NullTime) *time.Time { + if !value.Valid { + return nil + } + t := value.Time + return &t +} diff --git a/registry/migrations/src/go/migrator_test.go b/registry/migrations/src/go/migrator_test.go new file mode 100644 index 0000000..d8c33ad --- /dev/null +++ b/registry/migrations/src/go/migrator_test.go @@ -0,0 +1,288 @@ +package migrations + +import ( + "context" + "database/sql" + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + "io" + "sort" + "strings" + "sync" + "testing" + "testing/fstest" + "time" +) + +var ( + fakeDriverOnce sync.Once + fakeStatesMu sync.Mutex + fakeStates = map[string]*fakeState{} +) + +type fakeState struct { + mu sync.Mutex + applied map[int64]AppliedMigration + execs []string + failOn string + commits int + rollbacks int +} + +type fakeDriver struct{} +type fakeConn struct{ state *fakeState } +type fakeTx struct{ state *fakeState } +type fakeRows struct { + columns []string + values [][]driver.Value + index int +} +type fakeResult int64 + +func openFakeDB(t *testing.T) (*sql.DB, *fakeState) { + t.Helper() + fakeDriverOnce.Do(func() { + sql.Register("scion_migrations_fake", fakeDriver{}) + }) + name := fmt.Sprintf("db_%d", time.Now().UnixNano()) + state := &fakeState{applied: map[int64]AppliedMigration{}} + fakeStatesMu.Lock() + fakeStates[name] = state + fakeStatesMu.Unlock() + db, err := sql.Open("scion_migrations_fake", name) + if err != nil { + t.Fatalf("open fake db: %v", err) + } + t.Cleanup(func() { + _ = db.Close() + fakeStatesMu.Lock() + delete(fakeStates, name) + fakeStatesMu.Unlock() + }) + return db, state +} + +func (fakeDriver) Open(name string) (driver.Conn, error) { + fakeStatesMu.Lock() + defer fakeStatesMu.Unlock() + state := fakeStates[name] + if state == nil { + return nil, errors.New("unknown fake db") + } + return &fakeConn{state: state}, nil +} + +func (c *fakeConn) Prepare(string) (driver.Stmt, error) { + return nil, errors.New("prepare not implemented") +} +func (c *fakeConn) Close() error { return nil } +func (c *fakeConn) Begin() (driver.Tx, error) { return &fakeTx{state: c.state}, nil } + +func (c *fakeConn) BeginTx(context.Context, driver.TxOptions) (driver.Tx, error) { + return &fakeTx{state: c.state}, nil +} + +func (c *fakeConn) ExecContext(_ context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + c.state.mu.Lock() + defer c.state.mu.Unlock() + if c.state.failOn != "" && strings.Contains(query, c.state.failOn) { + return nil, errors.New("forced execution failure") + } + upper := strings.ToUpper(strings.TrimSpace(query)) + switch { + case strings.HasPrefix(upper, "CREATE TABLE IF NOT EXISTS"): + return fakeResult(0), nil + case strings.HasPrefix(upper, "INSERT INTO"): + version := args[0].Value.(int64) + c.state.applied[version] = AppliedMigration{ + Version: version, + Name: args[1].Value.(string), + Checksum: args[2].Value.(string), + AppliedAt: sql.NullTime{Time: args[3].Value.(time.Time), Valid: true}, + } + return fakeResult(1), nil + case strings.HasPrefix(upper, "DELETE FROM"): + delete(c.state.applied, args[0].Value.(int64)) + return fakeResult(1), nil + default: + c.state.execs = append(c.state.execs, query) + return fakeResult(1), nil + } +} + +func (c *fakeConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) { + c.state.mu.Lock() + defer c.state.mu.Unlock() + if !strings.HasPrefix(strings.ToUpper(strings.TrimSpace(query)), "SELECT VERSION") { + return nil, errors.New("unexpected query") + } + versions := make([]int64, 0, len(c.state.applied)) + for version := range c.state.applied { + versions = append(versions, version) + } + sort.Slice(versions, func(i, j int) bool { return versions[i] < versions[j] }) + values := make([][]driver.Value, 0, len(versions)) + for _, version := range versions { + row := c.state.applied[version] + values = append(values, []driver.Value{row.Version, row.Name, row.Checksum, row.AppliedAt.Time}) + } + return &fakeRows{ + columns: []string{"version", "name", "checksum", "applied_at"}, + values: values, + }, nil +} + +func (tx *fakeTx) Commit() error { + tx.state.mu.Lock() + defer tx.state.mu.Unlock() + tx.state.commits++ + return nil +} + +func (tx *fakeTx) Rollback() error { + tx.state.mu.Lock() + defer tx.state.mu.Unlock() + tx.state.rollbacks++ + return nil +} + +func (r *fakeRows) Columns() []string { return r.columns } +func (r *fakeRows) Close() error { return nil } +func (r *fakeRows) Next(dest []driver.Value) error { + if r.index >= len(r.values) { + return io.EOF + } + copy(dest, r.values[r.index]) + r.index++ + return nil +} + +func (fakeResult) LastInsertId() (int64, error) { return 0, nil } +func (r fakeResult) RowsAffected() (int64, error) { + return int64(r), nil +} + +func TestUpAppliesUnappliedMigrations(t *testing.T) { + db, state := openFakeDB(t) + fsys := fstest.MapFS{ + "20260101000001_add_users.up.sql": {Data: []byte("CREATE TABLE users(id BIGINT);")}, + "20260101000001_add_users.down.sql": {Data: []byte("DROP TABLE users;")}, + } + m, err := New(fsys, Options{Now: func() time.Time { return time.Unix(1, 0) }}) + if err != nil { + t.Fatalf("new: %v", err) + } + if err := m.Up(context.Background(), db); err != nil { + t.Fatalf("up: %v", err) + } + if len(state.applied) != 1 || len(state.execs) != 1 { + t.Fatalf("unexpected state: %+v", state) + } + if state.commits != 1 || state.rollbacks != 0 { + t.Fatalf("transaction counts: commits=%d rollbacks=%d", state.commits, state.rollbacks) + } +} + +func TestUpDetectsChecksumMismatch(t *testing.T) { + db, state := openFakeDB(t) + state.applied[20260101000001] = AppliedMigration{Version: 20260101000001, Name: "add_users", Checksum: "old"} + fsys := fstest.MapFS{ + "20260101000001_add_users.up.sql": {Data: []byte("CREATE TABLE users(id BIGINT);")}, + } + m, err := New(fsys) + if err != nil { + t.Fatalf("new: %v", err) + } + if err := m.Up(context.Background(), db); err == nil { + t.Fatalf("expected checksum mismatch") + } +} + +func TestDownRollsBackLatestMigration(t *testing.T) { + db, state := openFakeDB(t) + fsys := fstest.MapFS{ + "20260101000001_add_users.up.sql": {Data: []byte("CREATE TABLE users(id BIGINT);")}, + "20260101000001_add_users.down.sql": {Data: []byte("DROP TABLE users;")}, + "20260101000002_add_widgets.up.sql": {Data: []byte("CREATE TABLE widgets(id BIGINT);")}, + "20260101000002_add_widgets.down.sql": {Data: []byte("DROP TABLE widgets;")}, + } + m, err := New(fsys) + if err != nil { + t.Fatalf("new: %v", err) + } + if err := m.Up(context.Background(), db); err != nil { + t.Fatalf("up: %v", err) + } + if err := m.Down(context.Background(), db, 1); err != nil { + t.Fatalf("down: %v", err) + } + if _, ok := state.applied[20260101000002]; ok { + t.Fatalf("latest migration still applied") + } + if _, ok := state.applied[20260101000001]; !ok { + t.Fatalf("older migration was removed") + } +} + +func TestStatusReportsMissingAndMismatch(t *testing.T) { + db, state := openFakeDB(t) + state.applied[20260101000002] = AppliedMigration{Version: 20260101000002, Name: "missing", Checksum: "abc"} + state.applied[20260101000001] = AppliedMigration{Version: 20260101000001, Name: "add_users", Checksum: "old"} + fsys := fstest.MapFS{ + "20260101000001_add_users.up.sql": {Data: []byte("CREATE TABLE users(id BIGINT);")}, + } + m, err := New(fsys) + if err != nil { + t.Fatalf("new: %v", err) + } + statuses, err := m.Status(context.Background(), db) + if err != nil { + t.Fatalf("status: %v", err) + } + if len(statuses) != 2 { + t.Fatalf("len = %d", len(statuses)) + } + if !statuses[0].ChecksumMismatch || !statuses[1].Missing { + t.Fatalf("unexpected statuses: %+v", statuses) + } +} + +func TestStatusOmitsAppliedAtForUnappliedMigration(t *testing.T) { + db, _ := openFakeDB(t) + fsys := fstest.MapFS{ + "20260101000001_add_users.up.sql": {Data: []byte("CREATE TABLE users(id BIGINT);")}, + } + m, err := New(fsys) + if err != nil { + t.Fatalf("new: %v", err) + } + statuses, err := m.Status(context.Background(), db) + if err != nil { + t.Fatalf("status: %v", err) + } + data, err := json.Marshal(statuses[0]) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if strings.Contains(string(data), "applied_at") { + t.Fatalf("applied_at should be omitted for unapplied migration: %s", data) + } +} + +func TestAppliedRowsRespectsMaxMigrations(t *testing.T) { + db, state := openFakeDB(t) + state.applied[20260101000001] = AppliedMigration{Version: 20260101000001, Name: "one", Checksum: "a"} + state.applied[20260101000002] = AppliedMigration{Version: 20260101000002, Name: "two", Checksum: "b"} + fsys := fstest.MapFS{ + "20260101000001_one.up.sql": {Data: []byte("SELECT 1;")}, + } + m, err := New(fsys, Options{MaxMigrations: 1}) + if err != nil { + t.Fatalf("new: %v", err) + } + if _, err := m.Status(context.Background(), db); err == nil { + t.Fatalf("expected applied row limit error") + } +} diff --git a/registry/migrations/src/go/pentest_test.go b/registry/migrations/src/go/pentest_test.go new file mode 100644 index 0000000..1b7e052 --- /dev/null +++ b/registry/migrations/src/go/pentest_test.go @@ -0,0 +1,48 @@ +package migrations + +import ( + "testing" + "testing/fstest" +) + +func TestPentestRejectsPathTraversalDirectory(t *testing.T) { + _, err := New(fstest.MapFS{}, Options{Dir: "../secrets"}) + if err == nil { + t.Fatalf("expected path traversal rejection") + } +} + +func TestPentestRejectsEmbeddedPathTraversalDirectory(t *testing.T) { + _, err := New(fstest.MapFS{}, Options{Dir: "db/../secrets"}) + if err == nil { + t.Fatalf("expected embedded path traversal rejection") + } +} + +func TestPentestRejectsCRLFInTableName(t *testing.T) { + _, err := New(fstest.MapFS{}, Options{TableName: "schema_migrations\r\nDROP"}) + if err == nil { + t.Fatalf("expected CRLF rejection") + } +} + +func TestPentestRejectsNullByteInSQL(t *testing.T) { + fsys := fstest.MapFS{ + "20260101000001_add_users.up.sql": {Data: []byte("SELECT 1;\x00")}, + } + if _, err := Load(fsys); err == nil { + t.Fatalf("expected null byte rejection") + } +} + +func TestPentestRejectsTooManyMigrationFiles(t *testing.T) { + fsys := fstest.MapFS{ + "20260101000001_a.up.sql": {Data: []byte("SELECT 1;")}, + "20260101000002_b.up.sql": {Data: []byte("SELECT 1;")}, + "20260101000003_c.up.sql": {Data: []byte("SELECT 1;")}, + } + _, err := Load(fsys, Options{MaxMigrations: 1}) + if err == nil { + t.Fatalf("expected max migration rejection") + } +} diff --git a/registry/problem/README.md b/registry/problem/README.md new file mode 100644 index 0000000..383cf8a --- /dev/null +++ b/registry/problem/README.md @@ -0,0 +1,56 @@ +# Problem Module + +RFC 9457-style HTTP API problem responses with safe defaults. + +## What's Included + +- `Problem` and `InvalidParam` JSON types +- `Write` helper for `application/problem+json` +- `Handler` adapter for handlers that return `error` +- Panic recovery middleware +- Request ID extension support +- CRLF, null byte, length, and validation-error count protection + +## Quick Copy + +```bash +cp -r registry/problem/src/go/*.go yourproject/internal/problem/ +``` + +Or with the Scion CLI: + +```bash +scion add problem --to internal/problem +``` + +## Usage + +```go +http.Handle("/users", problem.Handler(func(w http.ResponseWriter, r *http.Request) error { + return problem.Error(http.StatusNotFound, "User not found", "no user matched the request") +})) +``` + +Validation errors: + +```go +problem.Write(w, r, problem.Validation([]problem.InvalidParam{ + {Detail: "must be a valid email", Pointer: "#/email"}, +})) +``` + +## File Reference + +| File | Purpose | +|------|---------| +| `config.go` | Options, defaults, env loading | +| `problem.go` | Problem types and sanitization | +| `handler.go` | JSON writer, error handler, recovery middleware | +| `pentest_test.go` | Attack-scenario tests | + +## Tests + +```bash +cd registry/problem/src/go +go test -v ./... +``` diff --git a/registry/problem/__llms__.md b/registry/problem/__llms__.md new file mode 100644 index 0000000..a6ed56f --- /dev/null +++ b/registry/problem/__llms__.md @@ -0,0 +1,3 @@ +# problem module + +Zero-dependency Go module for RFC 9457-style API errors. Copy `src/go/*.go` into `internal/problem`. Use `Write(w, r, Problem)`, `Error(status,title,detail)`, `Handler(func(w,r) error)`, and `Recoverer()`. Responses use `application/problem+json` and sanitize type/title/detail/instance/request_id/validation errors for CRLF, null bytes, length limits, and internal-error leakage. diff --git a/registry/problem/src/go/config.go b/registry/problem/src/go/config.go new file mode 100644 index 0000000..79201de --- /dev/null +++ b/registry/problem/src/go/config.go @@ -0,0 +1,93 @@ +// Package problem writes RFC 9457-style HTTP API error responses. +package problem + +import ( + "os" + "strconv" + "strings" +) + +// Options controls response sanitization and metadata. +type Options struct { + // TypeBase is prepended to relative problem types such as "validation". + TypeBase string + // MaxDetailLen caps Problem.Detail. + MaxDetailLen int + // MaxErrors caps validation error entries. + MaxErrors int + // MaxFieldLen caps all other string fields. + MaxFieldLen int + // IncludeRequestID copies a safe request ID into the response extensions. + IncludeRequestID bool + // RequestIDHeader names the request ID header to read when IncludeRequestID + // is true. + RequestIDHeader string +} + +// Defaults returns safe defaults for public API errors. +func Defaults() Options { + return Options{ + MaxDetailLen: 1024, + MaxErrors: 32, + MaxFieldLen: 256, + RequestIDHeader: "X-Request-ID", + } +} + +// FromEnv reads options from environment variables. +// +// Supported variables: +// - PROBLEM_TYPE_BASE +// - PROBLEM_MAX_DETAIL_LEN +// - PROBLEM_MAX_ERRORS +// - PROBLEM_INCLUDE_REQUEST_ID +// - PROBLEM_REQUEST_ID_HEADER +func FromEnv() Options { + o := Defaults() + if v := os.Getenv("PROBLEM_TYPE_BASE"); v != "" { + o.TypeBase = v + } + if v := os.Getenv("PROBLEM_MAX_DETAIL_LEN"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + o.MaxDetailLen = n + } + } + if v := os.Getenv("PROBLEM_MAX_ERRORS"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + o.MaxErrors = n + } + } + if v := os.Getenv("PROBLEM_INCLUDE_REQUEST_ID"); v != "" { + o.IncludeRequestID = strings.EqualFold(v, "true") + } + if v := os.Getenv("PROBLEM_REQUEST_ID_HEADER"); v != "" { + o.RequestIDHeader = v + } + return o +} + +func (o Options) normalize() Options { + d := Defaults() + if o.MaxDetailLen <= 0 { + o.MaxDetailLen = d.MaxDetailLen + } + if o.MaxDetailLen > 8192 { + o.MaxDetailLen = 8192 + } + if o.MaxErrors <= 0 { + o.MaxErrors = d.MaxErrors + } + if o.MaxErrors > 256 { + o.MaxErrors = 256 + } + if o.MaxFieldLen <= 0 { + o.MaxFieldLen = d.MaxFieldLen + } + if o.MaxFieldLen > 1024 { + o.MaxFieldLen = 1024 + } + if o.RequestIDHeader == "" { + o.RequestIDHeader = d.RequestIDHeader + } + return o +} diff --git a/registry/problem/src/go/config_test.go b/registry/problem/src/go/config_test.go new file mode 100644 index 0000000..be3d5d2 --- /dev/null +++ b/registry/problem/src/go/config_test.go @@ -0,0 +1,26 @@ +package problem + +import "testing" + +func TestOptionsNormalize(t *testing.T) { + opts := (Options{}).normalize() + if opts.MaxDetailLen != 1024 || opts.MaxErrors != 32 || opts.RequestIDHeader != "X-Request-ID" { + t.Fatalf("unexpected defaults: %+v", opts) + } +} + +func TestFromEnv(t *testing.T) { + t.Setenv("PROBLEM_TYPE_BASE", "https://api.example.com/problems") + t.Setenv("PROBLEM_MAX_DETAIL_LEN", "128") + t.Setenv("PROBLEM_MAX_ERRORS", "4") + t.Setenv("PROBLEM_INCLUDE_REQUEST_ID", "true") + t.Setenv("PROBLEM_REQUEST_ID_HEADER", "X-Trace-ID") + + opts := FromEnv().normalize() + if opts.TypeBase != "https://api.example.com/problems" || opts.MaxDetailLen != 128 { + t.Fatalf("unexpected env options: %+v", opts) + } + if opts.MaxErrors != 4 || !opts.IncludeRequestID || opts.RequestIDHeader != "X-Trace-ID" { + t.Fatalf("unexpected env options: %+v", opts) + } +} diff --git a/registry/problem/src/go/go.mod b/registry/problem/src/go/go.mod new file mode 100644 index 0000000..b5e5535 --- /dev/null +++ b/registry/problem/src/go/go.mod @@ -0,0 +1,3 @@ +module problem + +go 1.22 diff --git a/registry/problem/src/go/handler.go b/registry/problem/src/go/handler.go new file mode 100644 index 0000000..3e7c5d0 --- /dev/null +++ b/registry/problem/src/go/handler.go @@ -0,0 +1,124 @@ +package problem + +import ( + "encoding/json" + "errors" + "net/http" +) + +// HTTPError wraps a Problem as an error value for Handler. +type HTTPError struct { + Problem Problem + Err error +} + +func (e *HTTPError) Error() string { + if e == nil { + return "" + } + if e.Err != nil { + return e.Err.Error() + } + if e.Problem.Title != "" { + return e.Problem.Title + } + return http.StatusText(e.Problem.Status) +} + +func (e *HTTPError) Unwrap() error { + if e == nil { + return nil + } + return e.Err +} + +// Error creates an error value that Handler renders as a problem response. +func Error(status int, title, detail string) error { + return &HTTPError{Problem: New(status, title, detail)} +} + +// Write writes a sanitized problem response. +func Write(w http.ResponseWriter, r *http.Request, p Problem, opts ...Options) { + opt := Defaults() + if len(opts) > 0 { + opt = opts[0] + } + opt = opt.normalize() + if opt.IncludeRequestID && r != nil { + p.RequestID = r.Header.Get(opt.RequestIDHeader) + } + p = sanitizeProblem(p, opt) + w.Header().Set("Content-Type", mediaType) + w.WriteHeader(p.Status) + _ = json.NewEncoder(w).Encode(p) +} + +// Handler converts returned errors into problem responses and recovers panics. +func Handler(fn func(http.ResponseWriter, *http.Request) error, opts ...Options) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + lw := &responseWriter{ResponseWriter: w} + defer func() { + if recover() != nil { + if !lw.wrote { + Write(lw, r, Internal(), opts...) + } + } + }() + if fn == nil { + Write(lw, r, Internal(), opts...) + return + } + if err := fn(lw, r); err != nil { + if lw.wrote { + return + } + var httpErr *HTTPError + if errors.As(err, &httpErr) { + Write(lw, r, httpErr.Problem, opts...) + return + } + Write(lw, r, Internal(), opts...) + } + }) +} + +// Recoverer returns middleware that converts panics into generic 500 problem +// responses. +func Recoverer(opts ...Options) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + lw := &responseWriter{ResponseWriter: w} + defer func() { + if recover() != nil { + if !lw.wrote { + Write(lw, r, Internal(), opts...) + } + } + }() + next.ServeHTTP(lw, r) + }) + } +} + +type responseWriter struct { + http.ResponseWriter + wrote bool +} + +func (w *responseWriter) WriteHeader(status int) { + if !w.wrote { + w.wrote = true + w.ResponseWriter.WriteHeader(status) + } +} + +func (w *responseWriter) Write(b []byte) (int, error) { + if !w.wrote { + w.WriteHeader(http.StatusOK) + } + return w.ResponseWriter.Write(b) +} + +func (w *responseWriter) Unwrap() http.ResponseWriter { + return w.ResponseWriter +} diff --git a/registry/problem/src/go/handler_test.go b/registry/problem/src/go/handler_test.go new file mode 100644 index 0000000..2dd14c6 --- /dev/null +++ b/registry/problem/src/go/handler_test.go @@ -0,0 +1,92 @@ +package problem + +import ( + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestWriteProblemResponse(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/users", nil) + req.Header.Set("X-Request-ID", "req-123") + rec := httptest.NewRecorder() + Write(rec, req, New(http.StatusNotFound, "Not found", "missing"), Options{IncludeRequestID: true}) + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d", rec.Code) + } + if rec.Header().Get("Content-Type") != mediaType { + t.Fatalf("content-type = %q", rec.Header().Get("Content-Type")) + } + body := rec.Body.String() + if !strings.Contains(body, `"request_id":"req-123"`) || !strings.Contains(body, `"detail":"missing"`) { + t.Fatalf("unexpected body: %s", body) + } +} + +func TestHandlerMapsHTTPError(t *testing.T) { + h := Handler(func(http.ResponseWriter, *http.Request) error { + return Error(http.StatusConflict, "Conflict", "already exists") + }) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil)) + if rec.Code != http.StatusConflict { + t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String()) + } +} + +func TestHandlerHidesUnknownErrors(t *testing.T) { + h := Handler(func(http.ResponseWriter, *http.Request) error { + return errors.New("database password leaked") + }) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil)) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d", rec.Code) + } + if strings.Contains(rec.Body.String(), "database password") { + t.Fatalf("leaked internal error: %s", rec.Body.String()) + } +} + +func TestRecovererConvertsPanic(t *testing.T) { + h := Recoverer()(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + panic("boom") + })) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil)) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d", rec.Code) + } +} + +func TestRecovererDoesNotWriteAfterCommittedHeader(t *testing.T) { + h := Recoverer()(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusCreated) + panic("boom") + })) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil)) + if rec.Code != http.StatusCreated { + t.Fatalf("status = %d", rec.Code) + } + if strings.Contains(rec.Body.String(), "Internal Server Error") { + t.Fatalf("problem body was appended after committed header: %s", rec.Body.String()) + } +} + +func TestHandlerDoesNotWriteErrorAfterCommittedHeader(t *testing.T) { + h := Handler(func(w http.ResponseWriter, _ *http.Request) error { + w.WriteHeader(http.StatusAccepted) + return Error(http.StatusConflict, "Conflict", "already exists") + }) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil)) + if rec.Code != http.StatusAccepted { + t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String()) + } + if strings.Contains(rec.Body.String(), "Conflict") { + t.Fatalf("problem body was appended after committed header: %s", rec.Body.String()) + } +} diff --git a/registry/problem/src/go/pentest_test.go b/registry/problem/src/go/pentest_test.go new file mode 100644 index 0000000..d450eeb --- /dev/null +++ b/registry/problem/src/go/pentest_test.go @@ -0,0 +1,47 @@ +package problem + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestPentestRejectsCRLFAndNullInProblemFields(t *testing.T) { + rec := httptest.NewRecorder() + Write(rec, httptest.NewRequest(http.MethodGet, "/", nil), Problem{ + Type: "https://example.com/problems/bad\r\nx", + Title: "Bad\r\nTitle", + Status: http.StatusBadRequest, + Detail: "detail\x00secret", + Instance: "/items/1\r\nLocation: evil", + }) + body := rec.Body.String() + if strings.ContainsAny(body, "\r\x00") { + t.Fatalf("unsafe bytes in response: %q", body) + } + if strings.Contains(body, "secret") || strings.Contains(body, "Location") { + t.Fatalf("unsafe detail leaked: %s", body) + } +} + +func TestPentestLimitsValidationErrors(t *testing.T) { + errors := make([]InvalidParam, 40) + for i := range errors { + errors[i] = InvalidParam{Detail: "bad", Pointer: "#/field"} + } + p := sanitizeProblem(Validation(errors), Options{MaxErrors: 3}) + if len(p.Errors) != 3 { + t.Fatalf("errors len = %d", len(p.Errors)) + } +} + +func TestPentestRequestIDMustBeSafe(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Header.Set("X-Request-ID", "abc\r\nSet-Cookie: bad") + rec := httptest.NewRecorder() + Write(rec, req, New(http.StatusBadRequest, "Bad request", "bad"), Options{IncludeRequestID: true}) + if strings.Contains(rec.Body.String(), "Set-Cookie") { + t.Fatalf("unsafe request id leaked: %s", rec.Body.String()) + } +} diff --git a/registry/problem/src/go/problem.go b/registry/problem/src/go/problem.go new file mode 100644 index 0000000..a83ca23 --- /dev/null +++ b/registry/problem/src/go/problem.go @@ -0,0 +1,182 @@ +package problem + +import ( + "net/http" + "net/url" + "strings" +) + +const mediaType = "application/problem+json" + +// InvalidParam describes one invalid request field using a JSON Pointer. +type InvalidParam struct { + Detail string `json:"detail"` + Pointer string `json:"pointer,omitempty"` +} + +// Problem is the JSON shape defined by RFC 9457 plus safe extension fields. +type Problem struct { + Type string `json:"type"` + Title string `json:"title"` + Status int `json:"status"` + Detail string `json:"detail,omitempty"` + Instance string `json:"instance,omitempty"` + Errors []InvalidParam `json:"errors,omitempty"` + RequestID string `json:"request_id,omitempty"` +} + +// New creates a problem response. Unsafe fields are sanitized by Write. +func New(status int, title, detail string) Problem { + return Problem{ + Type: "about:blank", + Title: title, + Status: status, + Detail: detail, + } +} + +// Validation creates a 422 validation problem with field-level details. +func Validation(errors []InvalidParam) Problem { + return Problem{ + Type: "validation-error", + Title: "Request validation failed", + Status: http.StatusUnprocessableEntity, + Errors: errors, + } +} + +// Internal creates a generic 500 problem that does not expose implementation +// details. +func Internal() Problem { + return Problem{ + Type: "about:blank", + Title: http.StatusText(http.StatusInternalServerError), + Status: http.StatusInternalServerError, + } +} + +func sanitizeProblem(p Problem, opts Options) Problem { + opts = opts.normalize() + status := p.Status + if status < 400 || status > 599 { + status = http.StatusInternalServerError + } + title := safeField(p.Title, opts.MaxFieldLen) + if title == "" { + title = http.StatusText(status) + } + if title == "" { + title = "HTTP error" + } + return Problem{ + Type: sanitizeType(p.Type, opts), + Title: title, + Status: status, + Detail: safeField(p.Detail, opts.MaxDetailLen), + Instance: sanitizeInstance(p.Instance, opts), + Errors: sanitizeErrors(p.Errors, opts), + RequestID: safeField(p.RequestID, opts.MaxFieldLen), + } +} + +func sanitizeType(value string, opts Options) string { + if containsUnsafe(value) { + return "about:blank" + } + value = strings.TrimSpace(value) + if value == "" { + return "about:blank" + } + if len(value) > opts.MaxFieldLen { + return "about:blank" + } + if value == "about:blank" { + return value + } + if strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://") { + if _, err := url.ParseRequestURI(value); err == nil { + return value + } + return "about:blank" + } + if strings.HasPrefix(value, "/") { + return value + } + if opts.TypeBase != "" && !containsUnsafe(opts.TypeBase) { + base := strings.TrimRight(opts.TypeBase, "/") + return base + "/" + strings.TrimLeft(value, "/") + } + return "/" + strings.TrimLeft(value, "/") +} + +func sanitizeInstance(value string, opts Options) string { + if containsUnsafe(value) { + return "" + } + value = strings.TrimSpace(value) + if value == "" { + return "" + } + if len(value) > opts.MaxFieldLen { + return "" + } + if strings.HasPrefix(value, "/") || strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://") { + return value + } + return "" +} + +func sanitizeErrors(errors []InvalidParam, opts Options) []InvalidParam { + if len(errors) == 0 { + return nil + } + limit := len(errors) + if limit > opts.MaxErrors { + limit = opts.MaxErrors + } + out := make([]InvalidParam, 0, limit) + for _, item := range errors[:limit] { + detail := safeField(item.Detail, opts.MaxFieldLen) + if detail == "" { + continue + } + pointer := safePointer(item.Pointer, opts.MaxFieldLen) + out = append(out, InvalidParam{Detail: detail, Pointer: pointer}) + } + return out +} + +func safeField(value string, maxLen int) string { + if containsUnsafe(value) { + return "" + } + value = strings.TrimSpace(value) + if value == "" { + return "" + } + if len(value) > maxLen { + return value[:maxLen] + } + return value +} + +func safePointer(value string, maxLen int) string { + if containsUnsafe(value) { + return "" + } + value = strings.TrimSpace(value) + if value == "" { + return "" + } + if len(value) > maxLen { + return "" + } + if value == "#" || strings.HasPrefix(value, "#/") || strings.HasPrefix(value, "/") { + return value + } + return "" +} + +func containsUnsafe(value string) bool { + return strings.ContainsAny(value, "\r\n\x00") +} diff --git a/registry/problem/src/go/problem_test.go b/registry/problem/src/go/problem_test.go new file mode 100644 index 0000000..c97a3ed --- /dev/null +++ b/registry/problem/src/go/problem_test.go @@ -0,0 +1,45 @@ +package problem + +import ( + "net/http" + "strings" + "testing" +) + +func TestSanitizeProblemDefaultsStatusAndTitle(t *testing.T) { + p := sanitizeProblem(New(200, "", ""), Defaults()) + if p.Status != http.StatusInternalServerError || p.Title != http.StatusText(http.StatusInternalServerError) { + t.Fatalf("unexpected problem: %+v", p) + } +} + +func TestValidationProblemSanitizesErrors(t *testing.T) { + p := Validation([]InvalidParam{ + {Detail: "required", Pointer: "#/email"}, + {Detail: "bad\r\nheader", Pointer: "#/name"}, + {Detail: "bad pointer", Pointer: "name"}, + }) + p = sanitizeProblem(p, Options{MaxErrors: 2}) + if p.Type != "/validation-error" || len(p.Errors) != 1 { + t.Fatalf("unexpected validation problem: %+v", p) + } + if p.Errors[0].Pointer != "#/email" { + t.Fatalf("unexpected pointer: %+v", p.Errors[0]) + } +} + +func TestTypeBaseApplied(t *testing.T) { + p := sanitizeProblem(Problem{Type: "quota", Status: http.StatusTooManyRequests, Title: "Too many"}, Options{ + TypeBase: "https://api.example.com/problems", + }) + if p.Type != "https://api.example.com/problems/quota" { + t.Fatalf("type = %q", p.Type) + } +} + +func TestSafeFieldTruncates(t *testing.T) { + got := safeField(strings.Repeat("a", 10), 4) + if got != "aaaa" { + t.Fatalf("got %q", got) + } +}