diff --git a/AGENTS.md b/AGENTS.md index 7e63984..3b50ad8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ ## Project Description -Scion is a copy-paste code library for Go backend development. It contains 11 self-contained modules in `registry/` — each is a standalone Go package. Modules are standard-library only by default; security-sensitive modules may be declared as `stdlibOnly:false` in `registry/index.json` and copied in standalone mode. Modules are meant to be copied into a user's project and adapted, not imported as a dependency. +Scion is a copy-paste code library for Go backend development. It contains 12 self-contained modules in `registry/` — each is a standalone Go package. Modules are standard-library only by default; security-sensitive modules may be declared as `stdlibOnly:false` in `registry/index.json` and copied in standalone mode. Modules are meant to be copied into a user's project and adapted, not imported as a dependency. ## Coding Standards @@ -63,7 +63,7 @@ cd registry//src/go && go test -v -count=1 ./... ```bash # PowerShell -$modules = @('middleware','auth','crud','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') +$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Pop-Location } ``` @@ -89,7 +89,7 @@ You are working on Scion, a copy-paste code library for Go backend development. Project location: Architecture: -- 11 modules in registry/ — each is a standalone Go package +- 12 modules in registry/ — each is a standalone Go package - Module path pattern: registry//src/go/ - Go 1.22+, standard library by default, gofmt mandatory @@ -125,9 +125,10 @@ Task: ``` scion/ -├── registry/ # 11 copy-paste modules +├── registry/ # 12 copy-paste modules │ ├── auth/ # JWT auth + bcrypt │ ├── crud/ # Generic CRUD + pagination +│ ├── database/ # database/sql helpers │ ├── middleware/ # 9 HTTP middlewares │ ├── rbac/ # Role-based access control │ ├── ratelimit/ # 3 rate limiting algorithms diff --git a/AGENTS_zh.md b/AGENTS_zh.md index 0a1976c..fe30e47 100644 --- a/AGENTS_zh.md +++ b/AGENTS_zh.md @@ -6,7 +6,7 @@ ## 项目描述 -Scion 是一个面向 Go 后端开发的复制粘贴代码库。`registry/` 目录下有 11 个自包含模块，每个都是独立的 Go 包。模块默认仅使用标准库；安全敏感模块可以在 `registry/index.json` 中显式标记为 `stdlibOnly:false`，并以 standalone 模式复制。模块的设计目的是被复制到用户项目中并适配，而非作为依赖导入。 +Scion 是一个面向 Go 后端开发的复制粘贴代码库。`registry/` 目录下有 12 个自包含模块，每个都是独立的 Go 包。模块默认仅使用标准库；安全敏感模块可以在 `registry/index.json` 中显式标记为 `stdlibOnly:false`，并以 standalone 模式复制。模块的设计目的是被复制到用户项目中并适配，而非作为依赖导入。 ## 编码标准 @@ -63,7 +63,7 @@ cd registry//src/go && go test -v -count=1 ./... ```bash # PowerShell -$modules = @('middleware','auth','crud','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') +$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Pop-Location } ``` @@ -89,7 +89,7 @@ foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Po 项目路径: 架构: -- registry/ 下有 11 个模块 — 每个是独立的 Go 包 +- registry/ 下有 12 个模块 — 每个是独立的 Go 包 - 模块路径模式: registry/<模块名>/src/go/ - Go 1.22+，默认仅使用标准库，gofmt 强制 @@ -125,9 +125,10 @@ foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Po ``` scion/ -├── registry/ # 11 个复制粘贴模块 +├── registry/ # 12 个复制粘贴模块 │ ├── auth/ # JWT 认证 + bcrypt │ ├── crud/ # 泛型 CRUD + 分页 +│ ├── database/ # database/sql 工具 │ ├── middleware/ # 9 个 HTTP 中间件 │ ├── rbac/ # 角色权限控制 │ ├── ratelimit/ # 3 种限流算法 diff --git a/README.md b/README.md index 1acb599..b5bd143 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ Backend modules such as auth, CRUD, file upload, and rate limiting share most of |--------|-------------|-------------------| | [auth](registry/auth/) | JWT email/password auth + bcrypt | Rate limiting, user enumeration prevention, JTI, aud/iss validation | | [crud](registry/crud/) | Generic CRUD with pagination | Sort/filter whitelist, SQL injection prevention, pagination ceiling | +| [database](registry/database/) | `database/sql` setup + transactions | DSN-safe errors, whitelisted SQL fragments, parameterized values | | [middleware](registry/middleware/) | Recovery, CORS, logging, timeout, etc. | CRLF injection prevention, trusted proxy, body size limit | | [rbac](registry/rbac/) | Role-based access control | Wildcard permissions, cycle detection, hierarchy inheritance | | [ratelimit](registry/ratelimit/) | Fixed window / sliding window / token bucket | Memory exhaustion protection, LRU eviction, key length limit | @@ -109,6 +110,7 @@ scion/ | |-- auth/ # Authentication module | |-- cache/ # TTL + LRU cache | |-- crud/ # CRUD operations module +| |-- database/ # database/sql helpers | |-- file-upload/ # File upload handler | |-- health/ # Health check probes | |-- mail/ # SMTP email sender @@ -143,7 +145,7 @@ go run ./cmd/scion doctor --strict Run tests for all registry modules in PowerShell: ```powershell -$modules = @('middleware','auth','crud','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') +$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Pop-Location } ``` diff --git a/README_zh.md b/README_zh.md index c82108d..aef946d 100644 --- a/README_zh.md +++ b/README_zh.md @@ -74,6 +74,7 @@ Get-FileHash .\scion_v0.1.3_windows_amd64.zip -Algorithm SHA256 |------|------|----------| | [auth](registry/auth/) | JWT 邮箱/密码认证 + bcrypt | 限流、用户枚举防护、JTI、aud/iss 校验 | | [crud](registry/crud/) | 泛型 CRUD + 分页 | 排序/过滤白名单、SQL 注入防护、分页上限 | +| [database](registry/database/) | `database/sql` 设置 + 事务 | DSN 安全错误、SQL 片段白名单、参数化值 | | [middleware](registry/middleware/) | Recovery、CORS、日志、超时等 | CRLF 注入防护、可信代理、请求体大小限制 | | [rbac](registry/rbac/) | 基于角色的访问控制 | 通配符权限、循环检测、层级继承 | | [ratelimit](registry/ratelimit/) | 固定窗口 / 滑动窗口 / 令牌桶 | 内存耗尽防护、LRU 驱逐、key 长度限制 | @@ -109,6 +110,7 @@ scion/ | |-- auth/ # 认证模块 | |-- cache/ # TTL + LRU 缓存 | |-- crud/ # CRUD 模块 +| |-- database/ # database/sql 工具 | |-- file-upload/ # 文件上传模块 | |-- health/ # 健康检查模块 | |-- mail/ # SMTP 邮件模块 @@ -143,7 +145,7 @@ go run ./cmd/scion doctor --strict PowerShell 中运行所有 registry 模块测试： ```powershell -$modules = @('middleware','auth','crud','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') +$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Pop-Location } ``` diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 2fee675..e5f74b3 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -37,6 +37,7 @@ export default defineConfig({ { text: 'Overview', link: '/modules/' }, { text: 'Auth', link: '/modules/auth' }, { text: 'CRUD', link: '/modules/crud' }, + { text: 'Database', link: '/modules/database' }, { text: 'Middleware', link: '/modules/middleware' }, { text: 'RBAC', link: '/modules/rbac' }, { text: 'Rate Limit', link: '/modules/ratelimit' }, @@ -85,6 +86,7 @@ export default defineConfig({ { text: '概览', link: '/zh/modules/' }, { text: 'Auth 认证', link: '/zh/modules/auth' }, { text: 'CRUD 增删改查', link: '/zh/modules/crud' }, + { text: 'Database 数据库', link: '/zh/modules/database' }, { text: 'Middleware 中间件', link: '/zh/modules/middleware' }, { text: 'RBAC 权限控制', link: '/zh/modules/rbac' }, { text: 'Rate Limit 限流', link: '/zh/modules/ratelimit' }, diff --git a/docs/contributing.md b/docs/contributing.md index 15323c4..8c5fa36 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -66,7 +66,7 @@ Add your module to `registry/index.json`. cd registry//src/go && go test -v ./... # Test all modules -$modules = @('middleware','auth','crud','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') +$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Pop-Location } ``` diff --git a/docs/index.md b/docs/index.md index ff901ff..e897177 100644 --- a/docs/index.md +++ b/docs/index.md @@ -25,6 +25,10 @@ features: title: CRUD details: Generic CRUD operations with pagination, sort/filter whitelist, SQL injection prevention. link: /modules/crud + - icon: SQL + title: Database + details: database/sql setup, transactions, and whitelisted query fragments. + link: /modules/database - icon: 🛡️ title: Middleware details: Recovery, CORS, logging, timeout, request ID, body size limit. diff --git a/docs/modules/database.md b/docs/modules/database.md new file mode 100644 index 0000000..619c486 --- /dev/null +++ b/docs/modules/database.md @@ -0,0 +1,91 @@ +# Database Module + +`database/sql` setup, transaction helpers, and safe query fragments. + +## What's Included + +- Bounded dynamic connection pool defaults +- Dynamic pool sizing based on CPU and IO capacity +- `FromEnv()` for `DATABASE_*` settings +- Startup ping with timeout and DSN-safe errors +- `WithinTx` helper with rollback on error or panic +- `DBTX` interface for repositories +- Whitelisted `WHERE` and `ORDER BY` fragment builders +- Maximum 32 filter conditions per builder/query + +## Quick Copy + +```bash +cp -r registry/database/src/go/* yourproject/internal/database/ +``` + +## Usage + +```go +opts := database.FromEnv() +db, err := database.Open(ctx, opts) +if err != nil { + return err +} +defer db.Close() + +err = database.WithinTx(ctx, db, nil, func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, "UPDATE users SET active = ? WHERE id = ?", true, 123) + return err +}) +``` + +## Safe Fragments + +```go +columns := database.ColumnMap{ + "email": "users.email", + "created_at": "users.created_at", +} + +where, args, err := database.WhereEqual( + map[string]string{"email": "ada@example.com"}, + columns, + database.DollarPlaceholder, +) +order, err := database.OrderBy("-created_at", columns) +``` + +## Dynamic Pool Sizing + +`Defaults()` uses a balanced dynamic pool based on `GOMAXPROCS`. For IO-heavy +services, opt into a larger bounded pool: + +```go +opts := database.ApplyPoolStrategy(database.FromEnv(), database.PoolStrategy{ + Profile: database.PoolIOHeavy, + CPUCores: 8, + IOParallelism: 4, + MaxOpenLimit: 96, +}) +``` + +Environment knobs: + +| Variable | Purpose | +|----------|---------| +| `DATABASE_POOL_PROFILE` | `conservative`, `balanced`, or `io-heavy` | +| `DATABASE_POOL_CPU_CORES` | CPU cores used by the formula | +| `DATABASE_IO_PARALLELISM` | extra independent IO capacity | +| `DATABASE_POOL_MAX_OPEN_LIMIT` | hard cap for derived max open connections | + +## Security Features + +- Rejects CRLF and null bytes in string inputs +- Enforces length limits on driver names, DSNs, fields, and values +- Requires whitelist mapping for SQL identifiers +- Keeps filter values parameterized +- Caps generated filter conditions to avoid unbounded memory growth +- Does not include DSNs in open or ping errors + +## Tests + +```bash +cd registry/database/src/go +go test -v ./... +``` diff --git a/docs/modules/index.md b/docs/modules/index.md index bf247a3..e23d752 100644 --- a/docs/modules/index.md +++ b/docs/modules/index.md @@ -1,6 +1,6 @@ # Modules Overview -Scion provides 11 production-ready, copy-paste Go modules. Each module is self-contained. Modules are standard-library only by default; declared security exceptions are marked in the registry. +Scion provides 12 production-ready, copy-paste Go modules. Each module is self-contained. Modules are standard-library only by default; declared security exceptions are marked in the registry. ## Available Modules @@ -8,6 +8,7 @@ Scion provides 11 production-ready, copy-paste Go modules. Each module is self-c |--------|-------------|-------------------| | [Auth](/modules/auth) | JWT email/password auth + bcrypt | Rate limiting, user enumeration prevention, JTI | | [CRUD](/modules/crud) | Generic CRUD with pagination | Sort/filter whitelist, SQL injection prevention | +| [Database](/modules/database) | `database/sql` setup + transactions | DSN-safe errors, whitelisted SQL fragments | | [Middleware](/modules/middleware) | Recovery, CORS, logging, timeout | CRLF injection prevention, body size limit | | [RBAC](/modules/rbac) | Role-based access control | Wildcard permissions, cycle detection | | [Rate Limit](/modules/ratelimit) | Fixed/sliding window, token bucket | Memory exhaustion protection, LRU eviction | diff --git a/docs/zh/contributing.md b/docs/zh/contributing.md index 261f9bb..32aa69a 100644 --- a/docs/zh/contributing.md +++ b/docs/zh/contributing.md @@ -66,7 +66,7 @@ go test -v ./... cd registry//src/go && go test -v ./... # 测试所有模块 -$modules = @('middleware','auth','crud','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') +$modules = @('middleware','auth','crud','database','rbac','ratelimit','validation','file-upload','health','cache','pagination','mail') foreach ($m in $modules) { Push-Location "registry/$m/src/go"; go test ./...; Pop-Location } ``` diff --git a/docs/zh/index.md b/docs/zh/index.md index 9c1e3e8..0f55717 100644 --- a/docs/zh/index.md +++ b/docs/zh/index.md @@ -25,6 +25,10 @@ features: title: CRUD 增删改查 details: 通用 CRUD 操作 + 分页，排序/过滤白名单，SQL注入防护。 link: /zh/modules/crud + - icon: SQL + title: Database 数据库 + details: database/sql 设置、事务封装和 SQL 片段白名单。 + link: /zh/modules/database - icon: 🛡️ title: Middleware 中间件 details: Recovery、CORS、日志、超时、请求ID、请求体大小限制。 diff --git a/docs/zh/modules/database.md b/docs/zh/modules/database.md new file mode 100644 index 0000000..c0d335e --- /dev/null +++ b/docs/zh/modules/database.md @@ -0,0 +1,91 @@ +# Database 数据库模块 + +`database/sql` 设置、事务封装和安全 SQL 片段构建。 + +## 包含内容 + +- 显式连接池默认值 +- 基于 CPU 和 IO 能力的动态连接池 sizing +- 基于 `DATABASE_*` 的 `FromEnv()` +- 带超时的启动 ping，错误不暴露 DSN +- `WithinTx`，在错误或 panic 时回滚 +- 供 repository 使用的 `DBTX` 接口 +- 基于白名单的 `WHERE` 和 `ORDER BY` 片段构建 +- 每个 builder/query 最多 32 个过滤条件 + +## 快速复制 + +```bash +cp -r registry/database/src/go/* yourproject/internal/database/ +``` + +## 使用方式 + +```go +opts := database.FromEnv() +db, err := database.Open(ctx, opts) +if err != nil { + return err +} +defer db.Close() + +err = database.WithinTx(ctx, db, nil, func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, "UPDATE users SET active = ? WHERE id = ?", true, 123) + return err +}) +``` + +## 安全 SQL 片段 + +```go +columns := database.ColumnMap{ + "email": "users.email", + "created_at": "users.created_at", +} + +where, args, err := database.WhereEqual( + map[string]string{"email": "ada@example.com"}, + columns, + database.DollarPlaceholder, +) +order, err := database.OrderBy("-created_at", columns) +``` + +## 动态连接池策略 + +`Defaults()` 默认使用基于 `GOMAXPROCS` 的 balanced 动态连接池。IO-heavy +服务可以显式使用更大的有界连接池： + +```go +opts := database.ApplyPoolStrategy(database.FromEnv(), database.PoolStrategy{ + Profile: database.PoolIOHeavy, + CPUCores: 8, + IOParallelism: 4, + MaxOpenLimit: 96, +}) +``` + +环境变量： + +| 变量 | 用途 | +|------|------| +| `DATABASE_POOL_PROFILE` | `conservative`、`balanced` 或 `io-heavy` | +| `DATABASE_POOL_CPU_CORES` | sizing 公式使用的 CPU 核数 | +| `DATABASE_IO_PARALLELISM` | 额外独立 IO 能力 | +| `DATABASE_POOL_MAX_OPEN_LIMIT` | 推导出的最大连接数硬上限 | + +## 安全特性 + +- 拒绝字符串输入中的 CRLF 和 null 字节 +- 对 driver 名称、DSN、字段和值设置长度限制 +- SQL 标识符必须来自白名单映射 +- 过滤值只作为参数返回，不拼接进 SQL +- 限制生成的过滤条件数量，避免无界内存增长 +- open 或 ping 错误不包含 DSN + +## 测试 + +```bash +cd registry/database/src/go +go test -v ./... +``` diff --git a/docs/zh/modules/index.md b/docs/zh/modules/index.md index b349ec9..92caba4 100644 --- a/docs/zh/modules/index.md +++ b/docs/zh/modules/index.md @@ -1,6 +1,6 @@ # 模块概览 -Scion 提供 11 个生产就绪、可复制粘贴的 Go 模块。每个模块自包含。模块默认仅使用标准库；安全例外会在 registry 中显式标记。 +Scion 提供 12 个生产就绪、可复制粘贴的 Go 模块。每个模块自包含。模块默认仅使用标准库；安全例外会在 registry 中显式标记。 ## 可用模块 @@ -8,6 +8,7 @@ Scion 提供 11 个生产就绪、可复制粘贴的 Go 模块。每个模块自 |------|------|---------| | [Auth](/zh/modules/auth) | JWT 认证 + bcrypt | 限流，用户枚举防护，JTI | | [CRUD](/zh/modules/crud) | 通用 CRUD + 分页 | 排序/过滤白名单，SQL注入防护 | +| [Database](/zh/modules/database) | `database/sql` 设置 + 事务 | DSN 安全错误，SQL 片段白名单 | | [Middleware](/zh/modules/middleware) | Recovery、CORS、日志、超时 | CRLF 注入防护，请求体大小限制 | | [RBAC](/zh/modules/rbac) | 基于角色的访问控制 | 通配符权限，循环检测 | | [Rate Limit](/zh/modules/ratelimit) | 固定/滑动窗口、令牌桶 | 内存耗尽防护，LRU 淘汰 | diff --git a/internal/bundle/manifest.json b/internal/bundle/manifest.json index 75ea502..0d13152 100644 --- a/internal/bundle/manifest.json +++ b/internal/bundle/manifest.json @@ -1,7 +1,7 @@ { "schemaVersion": 1, - "registryVersion": "0.1.0", - "bundleHash": "18284db2be7ca31829f14a922988ce14aab2f5a2f224077c8e6ad7157ac7e003", + "registryVersion": "0.2.0", + "bundleHash": "a02b457e62e7e559ecef0dff6c0aeb47e901c0092099f1e83085aeb9ddc8c3a6", "modules": [ { "id": "auth", @@ -18,6 +18,11 @@ "version": "0.1.0", "source": "registry/crud/src/go" }, + { + "id": "database", + "version": "0.1.0", + "source": "registry/database/src/go" + }, { "id": "file-upload", "version": "0.1.0", @@ -113,15 +118,15 @@ "path": "registry/auth/src/go/handlers.go", "module": "auth", "moduleVersion": "0.1.0", - "sha256": "552c82e00c6eb87771f39987bf3e3bf4aec162b76a676f1aed01c545b4d80b06", - "size": 6555 + "sha256": "eadff0ccce281b58a7006f4cc5e84f3b54e4f880b1f4628ab86d5775505ab264", + "size": 6643 }, { "path": "registry/auth/src/go/handlers_test.go", "module": "auth", "moduleVersion": "0.1.0", - "sha256": "b0b29d3b202902b9da2ba4dee70981f32c96664ca593fa08633ec905ff473028", - "size": 10671 + "sha256": "188362f3741384dacf1631ac5f110c1578b0adbd8de87c105eb4bedb34c68623", + "size": 11023 }, { "path": "registry/auth/src/go/jwt.go", @@ -316,15 +321,15 @@ "path": "registry/crud/src/go/handlers.go", "module": "crud", "moduleVersion": "0.1.0", - "sha256": "9693b7588a165f4c218be244fb456d72e014ea9ab539b1eb51ed886affed0793", - "size": 6610 + "sha256": "b17cd5d67b8eac99c7f406a8ac90ddb87296ee6bbf9b815eae2a014e946da705", + "size": 7439 }, { "path": "registry/crud/src/go/handlers_test.go", "module": "crud", "moduleVersion": "0.1.0", - "sha256": "de92b97b818cb0a64c5203399d1c1f5c222893187b7eb6e439e6b5f025e331cf", - "size": 10024 + "sha256": "07248e7f6421e02182dcedaf5bbf1beb13565ee81d0538984292f6145cd21ed7", + "size": 11705 }, { "path": "registry/crud/src/go/models.go", @@ -344,8 +349,8 @@ "path": "registry/crud/src/go/pentest_test.go", "module": "crud", "moduleVersion": "0.1.0", - "sha256": "d971b4ef9942d82ae8e1a5825e67d9925577f8c6f079180c061667dd2bd433e5", - "size": 7847 + "sha256": "8c10b6eaaabadd67a2fa6bacf838af94ddb965c1a4918fec480baaa463de1ded", + "size": 8462 }, { "path": "registry/crud/src/go/routes.go", @@ -361,6 +366,111 @@ "sha256": "962e48fd141f103df2a6648e3232c4b796a9e2356ad3adcf88b6ecdf946d7713", "size": 3052 }, + { + "path": "registry/database/README.md", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "5ed6a2fe853d535b0f742b8ba0f9cb34b1ac06344946b3177e555757edcdeb81", + "size": 3005 + }, + { + "path": "registry/database/__llms__.md", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "6b2ea5f58fa15f14cc95830b33695cc24fcd0bbc7a25f580a108eea32dbbccac", + "size": 491 + }, + { + "path": "registry/database/src/go/config.go", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "35fceebcf600899f9b34f2bb8296a27343c7d1edb61f3e3ed0f94863172ce955", + "size": 6588 + }, + { + "path": "registry/database/src/go/config_test.go", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "54658ff8144fbc31589ed1f159ee68eb9d3e9dcbcde464161067a887ee06dc58", + "size": 4491 + }, + { + "path": "registry/database/src/go/driver_test.go", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "00a03b8f9cbc21328daef9624a37ace812fa91c5abfa04d07fbc20e9b63602f8", + "size": 2461 + }, + { + "path": "registry/database/src/go/go.mod", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "c5e7df7c136894507b1fe57e2e2f172c279d43bf0e1dc85187192b11034c3214", + "size": 25 + }, + { + "path": "registry/database/src/go/open.go", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "4177178b6b7b6c8093fbb11c79eb65b8b00c50447daa4f486a25c6b5af0273cf", + "size": 1000 + }, + { + "path": "registry/database/src/go/open_test.go", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "a808038e1ebaadf6830c76971ab8c455aaece0d08f54fb0679913145d2a80a99", + "size": 2054 + }, + { + "path": "registry/database/src/go/pentest_test.go", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "6b50d4b74b34a8469a857f95086895f4bb0b66ddce74588c0f6c4bb66ca312c9", + "size": 2874 + }, + { + "path": "registry/database/src/go/profile.go", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "2e4390e4ad1c17536cb50ed824f9847d4e317d11fbfa20941059b4923641bdc8", + "size": 2828 + }, + { + "path": "registry/database/src/go/profile_test.go", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "dfdc3188d4b8528a6ad0b60d4dc87c1a7f389455c7b04607f6274c7fa2128460", + "size": 2203 + }, + { + "path": "registry/database/src/go/query.go", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "599296ec3b5a1362635377a3508d972d8a81249256f35b016c7745287ef2a1f7", + "size": 6211 + }, + { + "path": "registry/database/src/go/query_test.go", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "85c05f1cbae1dab1d2f5799f0e3139b461631d5d0bcde6a3062d4c6cb07d28f9", + "size": 5832 + }, + { + "path": "registry/database/src/go/transaction.go", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "2888af8693a78d802183aece1b2ac16585197ebc258b17836938b07ebecbb0e6", + "size": 1841 + }, + { + "path": "registry/database/src/go/transaction_test.go", + "module": "database", + "moduleVersion": "0.1.0", + "sha256": "533397ceffc2a3a8f936ff42dc724db6b45be4552da47777faee93ef99daef80", + "size": 2823 + }, { "path": "registry/file-upload/README.md", "module": "file-upload", @@ -421,15 +531,15 @@ "path": "registry/file-upload/src/go/storage.go", "module": "file-upload", "moduleVersion": "0.1.0", - "sha256": "311c0244f5d12942048e892a7bcce92b1e8c3d25f5c7715194fb1823f1623cfa", - "size": 5572 + "sha256": "7d9be5c746bf1c2fa7e0c2d5f428aa4570ffdd961243cb790259326d754c7fd0", + "size": 7118 }, { "path": "registry/file-upload/src/go/storage_test.go", "module": "file-upload", "moduleVersion": "0.1.0", - "sha256": "a1b57a3ba9c83933e4c780b98402e0b571b8205d76a67accd184a8abb2be886a", - "size": 1265 + "sha256": "78309929f161124c981fd293beb246cb04e1a480a9aa29badaffca0f05f120a6", + "size": 2211 }, { "path": "registry/file-upload/src/go/validate.go", @@ -517,8 +627,8 @@ }, { "path": "registry/index.json", - "sha256": "535534825c49a7ee784b67e65b7d9d74b79e89f5b102739e61e96ab0633f6452", - "size": 6856 + "sha256": "529693ad0317b61ed6c347ee0ed37fb3d4a186716afae194fb31bf76c8510362", + "size": 7464 }, { "path": "registry/mail/README.md", @@ -650,8 +760,8 @@ "path": "registry/middleware/src/go/config.go", "module": "middleware", "moduleVersion": "0.1.0", - "sha256": "9e4c17317971a22e9ebc37f9ee39b620b3fa86f43f9014e543d96ae890e3dbe4", - "size": 5460 + "sha256": "058cf100feb21b79a1c842d26e61568a1d817a8a53bcdcc078bc880f2a2b5ec2", + "size": 5556 }, { "path": "registry/middleware/src/go/config_test.go", @@ -734,15 +844,15 @@ "path": "registry/middleware/src/go/proxy.go", "module": "middleware", "moduleVersion": "0.1.0", - "sha256": "076abf946207c97047c2d0bc979d3550687ba5c4f394f79a561efe33e873953a", - "size": 4624 + "sha256": "a4017e82ffb0487f99fb7abe7aa9c69d9ada8485e6aa454c2a687961e69b6ab9", + "size": 4854 }, { "path": "registry/middleware/src/go/proxy_test.go", "module": "middleware", "moduleVersion": "0.1.0", - "sha256": "7389c8876c91bc8581f89791b28afccdb5372d67a1ffc383ac20e465196e72ac", - "size": 4051 + "sha256": "6a831f91796d8e3cb09ec4ccf7cc45abbbee056e27ba471b9c1e27bfdbe2c74a", + "size": 5076 }, { "path": "registry/middleware/src/go/recovery.go", @@ -1049,15 +1159,15 @@ "path": "registry/rbac/src/go/manager.go", "module": "rbac", "moduleVersion": "0.1.0", - "sha256": "a3a73effe29751c7920589d1d9d97d28a3439a03df7d8abf1ff9ef579ef9ba8f", - "size": 7109 + "sha256": "0d8e1c9adb45f190a4b1f5ad345dc5f6bd4e33e100ac11876d8237486201fe3c", + "size": 7616 }, { "path": "registry/rbac/src/go/manager_test.go", "module": "rbac", "moduleVersion": "0.1.0", - "sha256": "ebc41319f5aa024ff615a53ede9c5ba6178638080d378dbe86bbf16955feccd3", - "size": 6170 + "sha256": "2a2646c190b4ad0ab6dc515ceb135c851af95b46841c9f961747752a0ddd16ca", + "size": 7641 }, { "path": "registry/rbac/src/go/middleware.go", diff --git a/internal/bundle/registry.zip b/internal/bundle/registry.zip index 860e6a8..e5f3887 100644 Binary files a/internal/bundle/registry.zip and b/internal/bundle/registry.zip differ diff --git a/registry/auth/src/go/handlers.go b/registry/auth/src/go/handlers.go index 6f1133c..6f7a043 100644 --- a/registry/auth/src/go/handlers.go +++ b/registry/auth/src/go/handlers.go @@ -63,11 +63,14 @@ func (h *Handler) WithLogger(logger *slog.Logger) *Handler { // decodeBody reads and size-limits the request body, then decodes JSON. func decodeBody(r *http.Request, dst interface{}) error { - body, err := io.ReadAll(io.LimitReader(r.Body, maxRequestBodySize)) + body, err := io.ReadAll(io.LimitReader(r.Body, maxRequestBodySize+1)) if err != nil { return err } defer r.Body.Close() + if len(body) > maxRequestBodySize { + return errors.New("request body too large") + } return json.Unmarshal(body, dst) } diff --git a/registry/auth/src/go/handlers_test.go b/registry/auth/src/go/handlers_test.go index e5908b3..2d5534d 100644 --- a/registry/auth/src/go/handlers_test.go +++ b/registry/auth/src/go/handlers_test.go @@ -332,7 +332,6 @@ func TestHandler_Me_Unauthorized(t *testing.T) { } func TestDecodeBody_MaxSize(t *testing.T) { - // 1MB + 1 should be truncated by LimitReader large := make([]byte, maxRequestBodySize+1) req := httptest.NewRequest(http.MethodPost, "/test", bytes.NewReader(large)) @@ -344,6 +343,17 @@ func TestDecodeBody_MaxSize(t *testing.T) { } } +func TestDecodeBody_RejectsOversizedValidJSONPrefix(t *testing.T) { + prefix := []byte(`{"key":"value"}`) + body := append(prefix, bytes.Repeat([]byte(" "), maxRequestBodySize-len(prefix)+1)...) + req := httptest.NewRequest(http.MethodPost, "/test", bytes.NewReader(body)) + + var dst map[string]interface{} + if err := decodeBody(req, &dst); err == nil { + t.Fatal("expected oversized body error") + } +} + func TestRespondJSON(t *testing.T) { rr := httptest.NewRecorder() respondJSON(rr, http.StatusOK, map[string]string{"key": "value"}) diff --git a/registry/crud/src/go/handlers.go b/registry/crud/src/go/handlers.go index b53811c..17b7e6d 100644 --- a/registry/crud/src/go/handlers.go +++ b/registry/crud/src/go/handlers.go @@ -2,10 +2,12 @@ package crud import ( "encoding/json" + "errors" "io" "log/slog" "net/http" "strconv" + "strings" ) // maxRequestBodySize limits request body to 1MB to prevent DoS. @@ -66,9 +68,16 @@ func (h *Handler[T]) WithFilterFields(allowed map[string]bool) *Handler[T] { return h } -// maxFilterValueLen limits the length of individual filter values to prevent -// abuse via excessively long query parameters. -const maxFilterValueLen = 256 +const ( + // maxFilterCount limits generated filter maps to prevent unbounded memory + // growth from requests with many unique query parameters. + maxFilterCount = 32 + // maxFilterKeyLen limits filter field names. + maxFilterKeyLen = 128 + // maxFilterValueLen limits the length of individual filter values to + // prevent abuse via excessively long query parameters. + maxFilterValueLen = 256 +) // Create handles POST requests. func (h *Handler[T]) Create(w http.ResponseWriter, r *http.Request) { @@ -122,26 +131,39 @@ func (h *Handler[T]) List(w http.ResponseWriter, r *http.Request) { return } - // Extract filter params + // Extract filter params. A nil allow-list disables filtering entirely. filter := make(map[string]string) - for key, values := range query { - if key == "offset" || key == "limit" || key == "sort" { - continue - } - if len(values) > 0 { - if h.filterAllowed != nil && !h.filterAllowed[key] { + if h.filterAllowed != nil { + for key, values := range query { + if key == "offset" || key == "limit" || key == "sort" { + continue + } + if len(filter) >= maxFilterCount { + respondError(w, http.StatusBadRequest, "too many filter fields") + return + } + if !isSafeFilterString(key, maxFilterKeyLen) { + respondError(w, http.StatusBadRequest, "invalid filter field") + return + } + if !h.filterAllowed[key] { respondError(w, http.StatusBadRequest, "invalid filter field: "+key) return } - // Limit filter value length to prevent abuse. + if len(values) == 0 { + continue + } val := values[0] - if len(val) > maxFilterValueLen { - respondError(w, http.StatusBadRequest, "filter value too long") + if !isSafeFilterString(val, maxFilterValueLen) { + respondError(w, http.StatusBadRequest, "invalid filter value") return } filter[key] = val } } + if len(filter) == 0 { + filter = nil + } params := ParseListParams(offset, limit, h.config.MaxPageSize, sort, filter) @@ -165,6 +187,13 @@ func (h *Handler[T]) List(w http.ResponseWriter, r *http.Request) { }) } +func isSafeFilterString(s string, maxLen int) bool { + if s == "" || len(s) > maxLen { + return false + } + return !strings.ContainsAny(s, "\r\n\x00") +} + // Update handles PUT requests by ID. func (h *Handler[T]) Update(w http.ResponseWriter, r *http.Request) { idStr := r.PathValue("id") @@ -218,11 +247,14 @@ func sortFieldFromRaw(raw string) string { // decodeBody reads and size-limits the request body, then decodes JSON. func decodeBody(r *http.Request, dst interface{}) error { - body, err := io.ReadAll(io.LimitReader(r.Body, maxRequestBodySize)) + body, err := io.ReadAll(io.LimitReader(r.Body, maxRequestBodySize+1)) if err != nil { return err } defer r.Body.Close() + if len(body) > maxRequestBodySize { + return errors.New("request body too large") + } return json.Unmarshal(body, dst) } diff --git a/registry/crud/src/go/handlers_test.go b/registry/crud/src/go/handlers_test.go index d30f777..56c9d53 100644 --- a/registry/crud/src/go/handlers_test.go +++ b/registry/crud/src/go/handlers_test.go @@ -6,6 +6,7 @@ import ( "errors" "net/http" "net/http/httptest" + "strconv" "strings" "testing" ) @@ -27,6 +28,7 @@ type mockEntityStore[T any] struct { deleteErr error listData []T listTotal int64 + lastList ListParams } func newMockEntityStore[T any]() *mockEntityStore[T] { @@ -51,7 +53,8 @@ func (m *mockEntityStore[T]) GetByID(id uint) (*T, error) { return e, nil } -func (m *mockEntityStore[T]) List(_ ListParams) ([]T, int64, error) { +func (m *mockEntityStore[T]) List(params ListParams) ([]T, int64, error) { + m.lastList = params if m.listErr != nil { return nil, 0, m.listErr } @@ -238,6 +241,24 @@ func TestHandler_List_FilterAllowed(t *testing.T) { } } +func TestHandler_List_FilterDisabledByDefault(t *testing.T) { + store := newMockEntityStore[Product]() + cfg := &Config{DefaultPageSize: 20, MaxPageSize: 100} + h := NewHandler(store, cfg) + + req := httptest.NewRequest(http.MethodGet, "/products?password=secret", nil) + rr := httptest.NewRecorder() + + h.List(rr, req) + + if rr.Code != http.StatusOK { + t.Errorf("expected status %d, got %d: %s", http.StatusOK, rr.Code, rr.Body.String()) + } + if store.lastList.Filter != nil { + t.Fatalf("expected filters to be disabled, got %#v", store.lastList.Filter) + } +} + func TestHandler_List_FilterDisallowed(t *testing.T) { store := newMockEntityStore[Product]() cfg := &Config{DefaultPageSize: 20, MaxPageSize: 100} @@ -256,6 +277,28 @@ func TestHandler_List_FilterDisallowed(t *testing.T) { } } +func TestHandler_List_RejectsTooManyFilters(t *testing.T) { + store := newMockEntityStore[Product]() + cfg := &Config{DefaultPageSize: 20, MaxPageSize: 100} + allowed := make(map[string]bool, maxFilterCount+1) + q := make([]string, 0, maxFilterCount+1) + for i := 0; i < maxFilterCount+1; i++ { + key := "f" + strconv.Itoa(i) + allowed[key] = true + q = append(q, key+"=x") + } + h := NewHandler(store, cfg).WithFilterFields(allowed) + + req := httptest.NewRequest(http.MethodGet, "/products?"+strings.Join(q, "&"), nil) + rr := httptest.NewRecorder() + + h.List(rr, req) + + if rr.Code != http.StatusBadRequest { + t.Errorf("expected status %d, got %d", http.StatusBadRequest, rr.Code) + } +} + func TestHandler_List_NilSliceProtection(t *testing.T) { store := newMockEntityStore[Product]() // listData is nil, listTotal is 0 @@ -371,6 +414,17 @@ func TestHandler_Delete_InvalidID(t *testing.T) { } } +func TestDecodeBodyRejectsOversizedValidJSONPrefix(t *testing.T) { + prefix := []byte(`{"name":"Widget"}`) + body := append(prefix, bytes.Repeat([]byte(" "), maxRequestBodySize-len(prefix)+1)...) + req := httptest.NewRequest(http.MethodPost, "/products", bytes.NewReader(body)) + + var dst Product + if err := decodeBody(req, &dst); err == nil { + t.Fatal("expected oversized body error") + } +} + func TestSortFieldFromRaw(t *testing.T) { if sortFieldFromRaw("name") != "name" { t.Error("expected name") diff --git a/registry/crud/src/go/pentest_test.go b/registry/crud/src/go/pentest_test.go index d06bc4b..8f43e01 100644 --- a/registry/crud/src/go/pentest_test.go +++ b/registry/crud/src/go/pentest_test.go @@ -182,6 +182,28 @@ func TestPentest_CRUD_FilterSQLInjection(t *testing.T) { } } +func TestPentest_CRUD_FilterCRLFAndNullRejected(t *testing.T) { + store := newMockEntityStore[pentestEntity]() + h := NewHandler[pentestEntity](store, &Config{MaxPageSize: 100, DefaultPageSize: 20}) + h.WithFilterFields(map[string]bool{"name": true}) + + tests := []string{ + "name=good%0D%0Abad", + "name=good%00bad", + "bad%0D%0Akey=value", + } + for _, rawQuery := range tests { + req := httptest.NewRequest(http.MethodGet, "/items?"+rawQuery, nil) + rec := httptest.NewRecorder() + + h.List(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Errorf("query %q: expected 400, got %d", rawQuery, rec.Code) + } + } +} + func TestPentest_CRUD_NegativeOffset(t *testing.T) { store := newMockEntityStore[pentestEntity]() h := NewHandler[pentestEntity](store, &Config{MaxPageSize: 100, DefaultPageSize: 20}) diff --git a/registry/database/README.md b/registry/database/README.md new file mode 100644 index 0000000..23067c3 --- /dev/null +++ b/registry/database/README.md @@ -0,0 +1,112 @@ +# Database Module + +Small `database/sql` helpers for connection setup, transaction wrapping, and +safe query fragments. + +## What's Included + +- Bounded dynamic `database/sql` pool defaults +- Dynamic pool sizing based on CPU and IO capacity +- Environment-based configuration +- Startup ping with timeout +- Transaction helper with rollback on error or panic +- Whitelisted `WHERE` and `ORDER BY` builders +- Filter condition cap to prevent unbounded slice growth +- `?` and PostgreSQL `$n` placeholders + +## Quick Copy + +```bash +cp -r registry/database/src/go/*.go yourproject/internal/database/ +``` + +Or with the Scion CLI: + +```bash +scion add database --to internal/database +``` + +## Usage + +```go +opts := database.FromEnv() +db, err := database.Open(ctx, opts) +if err != nil { + return err +} +defer db.Close() + +err = database.WithinTx(ctx, db, nil, func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, "UPDATE users SET active = ? WHERE id = ?", true, 123) + return err +}) +``` + +Safe fragments: + +```go +columns := database.ColumnMap{ + "email": "users.email", + "created_at": "users.created_at", +} + +where, args, err := database.WhereEqual( + map[string]string{"email": "ada@example.com"}, + columns, + database.DollarPlaceholder, +) +order, err := database.OrderBy("-created_at", columns) +_ = where +_ = args +_ = order +_ = err +``` + +Dynamic pool sizing: + +```go +opts := database.FromEnv() // balanced dynamic pool by default +opts = database.ApplyPoolStrategy(opts, database.PoolStrategy{ + Profile: database.PoolIOHeavy, + CPUCores: 8, + IOParallelism: 4, + MaxOpenLimit: 96, +}) +``` + +`Builder` is intended for one query. If you reuse it, call `Reset()` to drop +old argument references. + +## Environment + +| Variable | Purpose | Default | +|----------|---------|---------| +| `DATABASE_DRIVER` | `database/sql` driver name | required | +| `DATABASE_DSN` | driver-specific DSN | required | +| `DATABASE_MAX_OPEN_CONNS` | explicit max open override | dynamic | +| `DATABASE_MAX_IDLE_CONNS` | explicit max idle override | dynamic | +| `DATABASE_POOL_PROFILE` | `conservative`, `balanced`, `io-heavy` | `balanced` | +| `DATABASE_POOL_CPU_CORES` | CPU cores for dynamic sizing | `GOMAXPROCS` | +| `DATABASE_IO_PARALLELISM` | extra IO capacity for dynamic sizing | 1 | +| `DATABASE_POOL_MAX_OPEN_LIMIT` | hard cap for dynamic max open | profile default | +| `DATABASE_CONN_MAX_LIFETIME` | max connection lifetime | 30m | +| `DATABASE_CONN_MAX_IDLE_TIME` | max idle time | 10m | +| `DATABASE_PING_TIMEOUT` | startup ping timeout | 5s | + +## File Reference + +| File | Purpose | +|------|---------| +| `config.go` | Options, defaults, environment loading | +| `profile.go` | Dynamic connection pool sizing | +| `open.go` | Open and ping a configured `*sql.DB` | +| `transaction.go` | Transaction helper and DBTX interface | +| `query.go` | Whitelisted SQL fragment helpers | +| `pentest_test.go` | Security and abuse-case tests | + +## Tests + +```bash +cd registry/database/src/go +go test -v ./... +``` diff --git a/registry/database/__llms__.md b/registry/database/__llms__.md new file mode 100644 index 0000000..6c3a5d7 --- /dev/null +++ b/registry/database/__llms__.md @@ -0,0 +1,3 @@ +# Database + +Zero-dependency Go `database/sql` helper module. Copy `src/go/*.go` into `internal/database`; callers import their own SQL driver. Provides `Options`, dynamic pool `PoolStrategy`, `Defaults`, `FromEnv`, `Open`, `WithinTx`, `DBTX`, whitelisted `OrderBy`/`WhereEqual`, `Builder`, and `?`/`$n` placeholders. Rejects CRLF/null bytes and overlong strings, caps filter conditions, never concatenates untrusted filter/sort input into SQL, and does not include DSNs in open/ping errors. diff --git a/registry/database/src/go/config.go b/registry/database/src/go/config.go new file mode 100644 index 0000000..bf18314 --- /dev/null +++ b/registry/database/src/go/config.go @@ -0,0 +1,221 @@ +// Package database provides small database/sql helpers for copy-paste Go +// backends: safe connection-pool setup, transaction wrapping, and whitelisted +// query fragment construction. +// +// The package uses only the Go standard library. It does not import any SQL +// driver; callers must register the driver they use in their application. +package database + +import ( + "errors" + "os" + "strconv" + "strings" + "time" +) + +const ( + maxDriverNameLen = 64 + maxDSNLen = 4096 +) + +// Options configures database/sql opening and pool behaviour. +type Options struct { + // DriverName is the database/sql driver name, such as "postgres", "mysql", + // or "sqlite3". The driver must be imported by the calling application. + DriverName string + // DSN is the driver-specific data source name. It is never included in + // errors returned by this module. + DSN string + // MaxOpenConns limits concurrently open connections. Defaults are derived + // dynamically from RuntimePoolStrategy(PoolBalanced). + MaxOpenConns int + // MaxIdleConns limits idle connections retained by database/sql. Defaults + // are derived dynamically from RuntimePoolStrategy(PoolBalanced). + MaxIdleConns int + // ConnMaxLifetime is the maximum lifetime of one connection. Defaults to 30m. + ConnMaxLifetime time.Duration + // ConnMaxIdleTime is the maximum idle lifetime of one connection. Defaults to 10m. + ConnMaxIdleTime time.Duration + // PingTimeout bounds the startup ping performed by Open. Defaults to 5s. + PingTimeout time.Duration +} + +// Defaults returns safe dynamic pool defaults. DriverName and DSN must still be +// supplied by the caller or loaded with FromEnv. +func Defaults() Options { + opts := Options{ + ConnMaxLifetime: 30 * time.Minute, + ConnMaxIdleTime: 10 * time.Minute, + PingTimeout: 5 * time.Second, + } + return ApplyPoolStrategy(opts, RuntimePoolStrategy(PoolBalanced)) +} + +// FromEnv returns Defaults() overridden by environment variables: +// +// DATABASE_DRIVER +// DATABASE_DSN +// DATABASE_MAX_OPEN_CONNS +// DATABASE_MAX_IDLE_CONNS +// DATABASE_CONN_MAX_LIFETIME +// DATABASE_CONN_MAX_IDLE_TIME +// DATABASE_PING_TIMEOUT +// DATABASE_POOL_PROFILE (conservative, balanced, io-heavy) +// DATABASE_POOL_CPU_CORES +// DATABASE_IO_PARALLELISM +// DATABASE_POOL_MAX_OPEN_LIMIT +// +// Invalid numeric or duration values are ignored and the default is retained. +func FromEnv() Options { + opts := Defaults() + maxOpenSet := false + maxIdleSet := false + explicitMaxOpen := 0 + explicitMaxIdle := 0 + + if v := os.Getenv("DATABASE_DRIVER"); v != "" { + opts.DriverName = v + } + if v := os.Getenv("DATABASE_DSN"); v != "" { + opts.DSN = v + } + if v := os.Getenv("DATABASE_MAX_OPEN_CONNS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + opts.MaxOpenConns = n + explicitMaxOpen = n + maxOpenSet = true + } + } + if v := os.Getenv("DATABASE_MAX_IDLE_CONNS"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n >= 0 { + opts.MaxIdleConns = n + explicitMaxIdle = n + maxIdleSet = true + } + } + if v := os.Getenv("DATABASE_CONN_MAX_LIFETIME"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d >= 0 { + opts.ConnMaxLifetime = d + } + } + if v := os.Getenv("DATABASE_CONN_MAX_IDLE_TIME"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d >= 0 { + opts.ConnMaxIdleTime = d + } + } + if v := os.Getenv("DATABASE_PING_TIMEOUT"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + opts.PingTimeout = d + } + } + + strategy := RuntimePoolStrategy(PoolBalanced) + strategySet := false + if profile := os.Getenv("DATABASE_POOL_PROFILE"); profile != "" { + strategy.Profile = PoolProfile(profile) + strategySet = true + } + if v := os.Getenv("DATABASE_POOL_CPU_CORES"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + strategy.CPUCores = n + strategySet = true + } + } + if v := os.Getenv("DATABASE_IO_PARALLELISM"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n >= 0 { + strategy.IOParallelism = n + strategySet = true + } + } + if v := os.Getenv("DATABASE_POOL_MAX_OPEN_LIMIT"); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + strategy.MaxOpenLimit = n + strategySet = true + } + } + if strategySet { + opts = ApplyPoolStrategy(opts, strategy) + } + opts = applyExplicitPoolOverrides(opts, maxOpenSet, explicitMaxOpen, maxIdleSet, explicitMaxIdle) + return opts +} + +// Validate enforces required fields, length limits, and CRLF / null-byte +// rejection for all string options. +func (o Options) Validate() error { + if err := validateRequiredString("driver name", o.DriverName, maxDriverNameLen); err != nil { + return err + } + if err := validateRequiredString("dsn", o.DSN, maxDSNLen); err != nil { + return err + } + if o.MaxOpenConns <= 0 { + return errors.New("database: max open connections must be positive") + } + if o.MaxIdleConns < 0 { + return errors.New("database: max idle connections cannot be negative") + } + if o.MaxIdleConns > o.MaxOpenConns { + return errors.New("database: max idle connections cannot exceed max open connections") + } + if o.ConnMaxLifetime < 0 { + return errors.New("database: connection max lifetime cannot be negative") + } + if o.ConnMaxIdleTime < 0 { + return errors.New("database: connection max idle time cannot be negative") + } + if o.PingTimeout <= 0 { + return errors.New("database: ping timeout must be positive") + } + return nil +} + +func (o Options) normalize() Options { + defaults := Defaults() + if o.MaxOpenConns == 0 { + o.MaxOpenConns = defaults.MaxOpenConns + if o.MaxIdleConns == 0 { + o.MaxIdleConns = defaults.MaxIdleConns + } + } + if o.ConnMaxLifetime == 0 { + o.ConnMaxLifetime = defaults.ConnMaxLifetime + } + if o.ConnMaxIdleTime == 0 { + o.ConnMaxIdleTime = defaults.ConnMaxIdleTime + } + if o.PingTimeout == 0 { + o.PingTimeout = defaults.PingTimeout + } + return o +} + +func applyExplicitPoolOverrides(opts Options, maxOpenSet bool, maxOpen int, maxIdleSet bool, maxIdle int) Options { + if maxOpenSet { + opts.MaxOpenConns = maxOpen + } + if maxIdleSet { + opts.MaxIdleConns = maxIdle + } else if maxOpenSet && opts.MaxIdleConns > opts.MaxOpenConns { + opts.MaxIdleConns = opts.MaxOpenConns + } + return opts +} + +func validateRequiredString(name, value string, maxLen int) error { + if value == "" { + return errors.New("database: " + name + " is required") + } + return validateString(name, value, maxLen) +} + +func validateString(name, value string, maxLen int) error { + if len(value) > maxLen { + return errors.New("database: " + name + " is too long") + } + if strings.ContainsAny(value, "\r\n\x00") { + return errors.New("database: " + name + " contains invalid characters") + } + return nil +} diff --git a/registry/database/src/go/config_test.go b/registry/database/src/go/config_test.go new file mode 100644 index 0000000..020301a --- /dev/null +++ b/registry/database/src/go/config_test.go @@ -0,0 +1,131 @@ +package database + +import ( + "strings" + "testing" + "time" +) + +func TestDefaults(t *testing.T) { + opts := Defaults() + want := ApplyPoolStrategy(Options{ + ConnMaxLifetime: 30 * time.Minute, + ConnMaxIdleTime: 10 * time.Minute, + PingTimeout: 5 * time.Second, + }, RuntimePoolStrategy(PoolBalanced)) + if opts.MaxOpenConns != want.MaxOpenConns { + t.Fatalf("MaxOpenConns = %d", opts.MaxOpenConns) + } + if opts.MaxIdleConns != want.MaxIdleConns { + t.Fatalf("MaxIdleConns = %d", opts.MaxIdleConns) + } + if opts.ConnMaxLifetime != 30*time.Minute { + t.Fatalf("ConnMaxLifetime = %s", opts.ConnMaxLifetime) + } + if opts.ConnMaxIdleTime != 10*time.Minute { + t.Fatalf("ConnMaxIdleTime = %s", opts.ConnMaxIdleTime) + } + if opts.PingTimeout != 5*time.Second { + t.Fatalf("PingTimeout = %s", opts.PingTimeout) + } +} + +func TestFromEnv(t *testing.T) { + t.Setenv("DATABASE_DRIVER", "scion") + t.Setenv("DATABASE_DSN", "memory") + t.Setenv("DATABASE_MAX_OPEN_CONNS", "7") + t.Setenv("DATABASE_MAX_IDLE_CONNS", "3") + t.Setenv("DATABASE_CONN_MAX_LIFETIME", "2m") + t.Setenv("DATABASE_CONN_MAX_IDLE_TIME", "30s") + t.Setenv("DATABASE_PING_TIMEOUT", "1s") + + opts := FromEnv() + if opts.DriverName != "scion" || opts.DSN != "memory" { + t.Fatalf("env driver/dsn not loaded: %#v", opts) + } + if opts.MaxOpenConns != 7 || opts.MaxIdleConns != 3 { + t.Fatalf("env pool not loaded: %#v", opts) + } + if opts.ConnMaxLifetime != 2*time.Minute || opts.ConnMaxIdleTime != 30*time.Second || opts.PingTimeout != time.Second { + t.Fatalf("env durations not loaded: %#v", opts) + } +} + +func TestFromEnvIgnoresInvalidValues(t *testing.T) { + defaults := Defaults() + t.Setenv("DATABASE_MAX_OPEN_CONNS", "-1") + t.Setenv("DATABASE_MAX_IDLE_CONNS", "-1") + t.Setenv("DATABASE_CONN_MAX_LIFETIME", "bad") + t.Setenv("DATABASE_CONN_MAX_IDLE_TIME", "-1s") + t.Setenv("DATABASE_PING_TIMEOUT", "0s") + + opts := FromEnv() + if opts.MaxOpenConns != defaults.MaxOpenConns || opts.MaxIdleConns != defaults.MaxIdleConns { + t.Fatalf("invalid ints should retain defaults: %#v", opts) + } + if opts.ConnMaxLifetime != 30*time.Minute || opts.ConnMaxIdleTime != 10*time.Minute || opts.PingTimeout != 5*time.Second { + t.Fatalf("invalid durations should retain defaults: %#v", opts) + } +} + +func TestFromEnvExplicitMaxOpenClampsDerivedIdle(t *testing.T) { + t.Setenv("DATABASE_POOL_PROFILE", "io-heavy") + t.Setenv("DATABASE_POOL_CPU_CORES", "4") + t.Setenv("DATABASE_IO_PARALLELISM", "4") + t.Setenv("DATABASE_MAX_OPEN_CONNS", "5") + + opts := FromEnv() + if opts.MaxOpenConns != 5 { + t.Fatalf("MaxOpenConns = %d", opts.MaxOpenConns) + } + if opts.MaxIdleConns != 5 { + t.Fatalf("MaxIdleConns = %d", opts.MaxIdleConns) + } +} + +func TestOptionsValidate(t *testing.T) { + opts := Defaults() + opts.DriverName = "scion" + opts.DSN = "memory" + if err := opts.Validate(); err != nil { + t.Fatalf("valid options: %v", err) + } + + tests := []struct { + name string + opts Options + }{ + {"missing driver", Options{DSN: "memory", MaxOpenConns: 1, PingTimeout: time.Second}}, + {"missing dsn", Options{DriverName: "scion", MaxOpenConns: 1, PingTimeout: time.Second}}, + {"crlf driver", Options{DriverName: "scion\r\nbad", DSN: "memory", MaxOpenConns: 1, PingTimeout: time.Second}}, + {"null dsn", Options{DriverName: "scion", DSN: "mem\x00ory", MaxOpenConns: 1, PingTimeout: time.Second}}, + {"long driver", Options{DriverName: strings.Repeat("a", maxDriverNameLen+1), DSN: "memory", MaxOpenConns: 1, PingTimeout: time.Second}}, + {"idle exceeds open", Options{DriverName: "scion", DSN: "memory", MaxOpenConns: 1, MaxIdleConns: 2, PingTimeout: time.Second}}, + {"bad ping", Options{DriverName: "scion", DSN: "memory", MaxOpenConns: 1, PingTimeout: 0}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.opts.Validate(); err == nil { + t.Fatal("expected validation error") + } + }) + } +} + +func TestNormalizeFillsZeroPoolValues(t *testing.T) { + opts := Options{DriverName: "scion", DSN: "memory"}.normalize() + defaults := Defaults() + if opts.MaxOpenConns != defaults.MaxOpenConns || opts.MaxIdleConns != defaults.MaxIdleConns || opts.PingTimeout != 5*time.Second { + t.Fatalf("normalize did not fill defaults: %#v", opts) + } +} + +func TestNormalizeAllowsZeroIdleWhenMaxOpenSet(t *testing.T) { + opts := Options{DriverName: "scion", DSN: "memory", MaxOpenConns: 8, MaxIdleConns: 0}.normalize() + if opts.MaxOpenConns != 8 { + t.Fatalf("MaxOpenConns = %d", opts.MaxOpenConns) + } + if opts.MaxIdleConns != 0 { + t.Fatalf("MaxIdleConns = %d", opts.MaxIdleConns) + } +} diff --git a/registry/database/src/go/driver_test.go b/registry/database/src/go/driver_test.go new file mode 100644 index 0000000..ff2b69d --- /dev/null +++ b/registry/database/src/go/driver_test.go @@ -0,0 +1,136 @@ +package database + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "io" + "strings" + "sync" + "sync/atomic" + "time" +) + +const testDriverName = "sciontest" + +var registerTestDriver sync.Once + +type txCounters struct { + commits atomic.Int64 + rollbacks atomic.Int64 +} + +var testTxCounters txCounters + +func ensureTestDriver() { + registerTestDriver.Do(func() { + sql.Register(testDriverName, testDriver{}) + }) +} + +func resetTxCounters() { + testTxCounters.commits.Store(0) + testTxCounters.rollbacks.Store(0) +} + +type testDriver struct{} + +func (testDriver) Open(name string) (driver.Conn, error) { + if strings.Contains(name, "open-error") { + return nil, errors.New("open failed for " + name) + } + return &testConn{dsn: name}, nil +} + +type testConn struct { + dsn string +} + +func (c *testConn) Prepare(string) (driver.Stmt, error) { + return testStmt{}, nil +} + +func (c *testConn) Close() error { + return nil +} + +func (c *testConn) Begin() (driver.Tx, error) { + return c.BeginTx(context.Background(), driver.TxOptions{}) +} + +func (c *testConn) Ping(ctx context.Context) error { + if strings.Contains(c.dsn, "ping-error") { + return errors.New("ping failed for " + c.dsn) + } + if strings.Contains(c.dsn, "slow") { + timer := time.NewTimer(time.Second) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } + } + return nil +} + +func (c *testConn) BeginTx(context.Context, driver.TxOptions) (driver.Tx, error) { + if strings.Contains(c.dsn, "begin-error") { + return nil, errors.New("begin failed") + } + return &testTx{dsn: c.dsn}, nil +} + +type testTx struct { + dsn string +} + +func (tx *testTx) Commit() error { + testTxCounters.commits.Add(1) + if strings.Contains(tx.dsn, "commit-error") { + return errors.New("commit failed") + } + return nil +} + +func (tx *testTx) Rollback() error { + testTxCounters.rollbacks.Add(1) + if strings.Contains(tx.dsn, "rollback-error") { + return errors.New("rollback failed") + } + return nil +} + +type testStmt struct{} + +func (testStmt) Close() error { + return nil +} + +func (testStmt) NumInput() int { + return -1 +} + +func (testStmt) Exec([]driver.Value) (driver.Result, error) { + return driver.RowsAffected(0), nil +} + +func (testStmt) Query([]driver.Value) (driver.Rows, error) { + return emptyRows{}, nil +} + +type emptyRows struct{} + +func (emptyRows) Columns() []string { + return nil +} + +func (emptyRows) Close() error { + return nil +} + +func (emptyRows) Next([]driver.Value) error { + return io.EOF +} diff --git a/registry/database/src/go/go.mod b/registry/database/src/go/go.mod new file mode 100644 index 0000000..c83f728 --- /dev/null +++ b/registry/database/src/go/go.mod @@ -0,0 +1,3 @@ +module database + +go 1.22 diff --git a/registry/database/src/go/open.go b/registry/database/src/go/open.go new file mode 100644 index 0000000..ac4bcc7 --- /dev/null +++ b/registry/database/src/go/open.go @@ -0,0 +1,38 @@ +package database + +import ( + "context" + "database/sql" + "errors" + "fmt" +) + +// Open validates Options, opens a database/sql handle, applies pool settings, +// and verifies connectivity with PingContext. Returned errors do not include +// the DSN. +func Open(ctx context.Context, opts Options) (*sql.DB, error) { + if ctx == nil { + return nil, errors.New("database: context is nil") + } + opts = opts.normalize() + if err := opts.Validate(); err != nil { + return nil, err + } + + db, err := sql.Open(opts.DriverName, opts.DSN) + if err != nil { + return nil, fmt.Errorf("database: open driver %q: %w", opts.DriverName, err) + } + db.SetMaxOpenConns(opts.MaxOpenConns) + db.SetMaxIdleConns(opts.MaxIdleConns) + db.SetConnMaxLifetime(opts.ConnMaxLifetime) + db.SetConnMaxIdleTime(opts.ConnMaxIdleTime) + + pingCtx, cancel := context.WithTimeout(ctx, opts.PingTimeout) + defer cancel() + if err := db.PingContext(pingCtx); err != nil { + _ = db.Close() + return nil, errors.New("database: ping failed") + } + return db, nil +} diff --git a/registry/database/src/go/open_test.go b/registry/database/src/go/open_test.go new file mode 100644 index 0000000..138a0da --- /dev/null +++ b/registry/database/src/go/open_test.go @@ -0,0 +1,88 @@ +package database + +import ( + "context" + "strings" + "testing" + "time" +) + +func TestOpenSuccess(t *testing.T) { + ensureTestDriver() + opts := Defaults() + opts.DriverName = testDriverName + opts.DSN = "memory" + opts.MaxOpenConns = 4 + opts.MaxIdleConns = 2 + + db, err := Open(context.Background(), opts) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + + if got := db.Stats().MaxOpenConnections; got != 4 { + t.Fatalf("MaxOpenConnections = %d", got) + } +} + +func TestOpenAppliesDefaultPoolValues(t *testing.T) { + ensureTestDriver() + db, err := Open(context.Background(), Options{ + DriverName: testDriverName, + DSN: "memory", + }) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer db.Close() + want := Defaults().MaxOpenConns + if got := db.Stats().MaxOpenConnections; got != want { + t.Fatalf("MaxOpenConnections = %d, want %d", got, want) + } +} + +func TestOpenValidationError(t *testing.T) { + ensureTestDriver() + _, err := Open(context.Background(), Options{DriverName: testDriverName, DSN: "bad\r\nvalue"}) + if err == nil { + t.Fatal("expected validation error") + } +} + +func TestOpenRejectsNilContext(t *testing.T) { + if _, err := Open(nil, Options{DriverName: testDriverName, DSN: "memory"}); err == nil { + t.Fatal("expected nil context error") + } +} + +func TestOpenPingErrorDoesNotLeakDSN(t *testing.T) { + ensureTestDriver() + dsn := "ping-error user=app password=supersecret" + _, err := Open(context.Background(), Options{ + DriverName: testDriverName, + DSN: dsn, + PingTimeout: time.Second, + }) + if err == nil { + t.Fatal("expected ping error") + } + if strings.Contains(err.Error(), "supersecret") || strings.Contains(err.Error(), dsn) { + t.Fatalf("error leaked DSN: %v", err) + } +} + +func TestOpenPingTimeout(t *testing.T) { + ensureTestDriver() + _, err := Open(context.Background(), Options{ + DriverName: testDriverName, + DSN: "slow", + PingTimeout: time.Millisecond, + }) + if err == nil { + t.Fatal("expected ping timeout") + } + if strings.Contains(err.Error(), "slow") { + t.Fatalf("error leaked DSN: %v", err) + } +} diff --git a/registry/database/src/go/pentest_test.go b/registry/database/src/go/pentest_test.go new file mode 100644 index 0000000..bc7f90b --- /dev/null +++ b/registry/database/src/go/pentest_test.go @@ -0,0 +1,107 @@ +package database + +import ( + "context" + "fmt" + "strings" + "testing" + "time" +) + +func TestPentestRejectsCRLFAndNullOptions(t *testing.T) { + bad := []Options{ + {DriverName: "postgres\r\nX-Header: x", DSN: "memory", MaxOpenConns: 1, PingTimeout: time.Second}, + {DriverName: "postgres", DSN: "host=localhost\x00password=secret", MaxOpenConns: 1, PingTimeout: time.Second}, + } + for _, opts := range bad { + if err := opts.Validate(); err == nil { + t.Fatal("expected invalid options") + } + } +} + +func TestPentestOpenDoesNotLeakSecrets(t *testing.T) { + ensureTestDriver() + dsn := "ping-error host=db user=app password=do-not-leak" + _, err := Open(context.Background(), Options{ + DriverName: testDriverName, + DSN: dsn, + PingTimeout: time.Second, + }) + if err == nil { + t.Fatal("expected error") + } + msg := err.Error() + if strings.Contains(msg, "do-not-leak") || strings.Contains(msg, dsn) { + t.Fatalf("secret leaked in error: %s", msg) + } +} + +func TestPentestOrderByInjectionRejected(t *testing.T) { + payloads := []string{ + "created_at DESC; DROP TABLE users", + "created_at--", + "created_at/*comment*/", + "created_at\r\nLIMIT 1", + "-created_at;SELECT 1", + } + for _, payload := range payloads { + if _, err := OrderBy(payload, testColumns()); err == nil { + t.Fatalf("payload accepted: %q", payload) + } + } +} + +func TestPentestFilterKeyInjectionRejected(t *testing.T) { + payloads := []string{ + "email OR 1=1", + "email;DROP TABLE users", + "email\x00", + strings.Repeat("a", maxFieldLen+1), + } + for _, payload := range payloads { + _, _, err := WhereEqual(map[string]string{payload: "x"}, testColumns(), QuestionPlaceholder) + if err == nil { + t.Fatalf("payload accepted: %q", payload) + } + } +} + +func TestPentestFilterValueIsParameterized(t *testing.T) { + payload := "x' OR '1'='1" + sql, args, err := WhereEqual(map[string]string{"email": payload}, testColumns(), DollarPlaceholder) + if err != nil { + t.Fatalf("WhereEqual: %v", err) + } + if strings.Contains(sql, payload) { + t.Fatalf("payload was concatenated into SQL: %s", sql) + } + if len(args) != 1 || args[0] != payload { + t.Fatalf("payload should be returned as arg: %#v", args) + } +} + +func TestPentestUnsafeDeveloperWhitelistRejected(t *testing.T) { + columns := ColumnMap{ + "email": "users.email) OR 1=1 --", + } + if _, err := OrderBy("email", columns); err == nil { + t.Fatal("unsafe whitelist value accepted") + } +} + +func TestPentestNilWhitelistRejected(t *testing.T) { + if _, err := OrderBy("email", nil); err == nil { + t.Fatal("nil whitelist accepted") + } +} + +func TestPentestTooManyFiltersRejectedBeforeAllocationGrowth(t *testing.T) { + filters := make(map[string]string, maxConditions+100) + for i := 0; i < maxConditions+100; i++ { + filters[fmt.Sprintf("field%d", i)] = "x" + } + if _, _, err := WhereEqual(filters, nil, QuestionPlaceholder); err == nil { + t.Fatal("oversized filters accepted") + } +} diff --git a/registry/database/src/go/profile.go b/registry/database/src/go/profile.go new file mode 100644 index 0000000..fb8a52b --- /dev/null +++ b/registry/database/src/go/profile.go @@ -0,0 +1,110 @@ +package database + +import ( + "runtime" + "strings" +) + +// PoolProfile selects a connection-pool sizing formula. +type PoolProfile string + +const ( + // PoolConservative keeps database pressure low for small apps and shared DBs. + PoolConservative PoolProfile = "conservative" + // PoolBalanced is a general-purpose profile for mixed CPU and IO work. + PoolBalanced PoolProfile = "balanced" + // PoolIOHeavy allows more in-flight queries when the app is often waiting + // on network/storage IO and the database can absorb the extra concurrency. + PoolIOHeavy PoolProfile = "io-heavy" +) + +// PoolStrategy describes how to derive pool sizes from CPU and IO capacity. +type PoolStrategy struct { + // Profile selects the sizing formula. Unknown values fall back to balanced. + Profile PoolProfile + // CPUCores is the effective CPU parallelism. Values <= 0 use GOMAXPROCS. + CPUCores int + // IOParallelism represents extra independent IO capacity, such as separate + // disks, database replicas, or known downstream query capacity. + IOParallelism int + // MaxOpenLimit caps the derived MaxOpenConns. Values <= 0 use the profile default. + MaxOpenLimit int +} + +// RuntimePoolStrategy returns a strategy using the process CPU parallelism. +func RuntimePoolStrategy(profile PoolProfile) PoolStrategy { + return PoolStrategy{ + Profile: profile, + CPUCores: runtime.GOMAXPROCS(0), + IOParallelism: 1, + } +} + +// ApplyPoolStrategy returns opts with MaxOpenConns and MaxIdleConns derived +// from strategy. Other Options fields are preserved. +func ApplyPoolStrategy(opts Options, strategy PoolStrategy) Options { + profile := normalizePoolProfile(strategy.Profile) + cores := strategy.CPUCores + if cores <= 0 { + cores = runtime.GOMAXPROCS(0) + } + if cores < 1 { + cores = 1 + } + ioParallelism := strategy.IOParallelism + if ioParallelism < 0 { + ioParallelism = 0 + } + + var maxOpen, defaultLimit int + var idleRatio int + switch profile { + case PoolConservative: + maxOpen = cores + ioParallelism + defaultLimit = 16 + idleRatio = 4 + case PoolIOHeavy: + maxOpen = cores*4 + ioParallelism*2 + defaultLimit = 128 + idleRatio = 2 + default: + maxOpen = cores*2 + ioParallelism + defaultLimit = 64 + idleRatio = 3 + } + + if maxOpen < 1 { + maxOpen = 1 + } + limit := strategy.MaxOpenLimit + if limit <= 0 { + limit = defaultLimit + } + if maxOpen > limit { + maxOpen = limit + } + + maxIdle := maxOpen / idleRatio + if maxIdle < 1 { + maxIdle = 1 + } + if maxIdle > maxOpen { + maxIdle = maxOpen + } + opts.MaxOpenConns = maxOpen + opts.MaxIdleConns = maxIdle + return opts +} + +func normalizePoolProfile(profile PoolProfile) PoolProfile { + switch PoolProfile(strings.ToLower(strings.TrimSpace(string(profile)))) { + case PoolConservative: + return PoolConservative + case PoolIOHeavy: + return PoolIOHeavy + case PoolBalanced: + return PoolBalanced + default: + return PoolBalanced + } +} diff --git a/registry/database/src/go/profile_test.go b/registry/database/src/go/profile_test.go new file mode 100644 index 0000000..5b6a965 --- /dev/null +++ b/registry/database/src/go/profile_test.go @@ -0,0 +1,82 @@ +package database + +import "testing" + +func TestApplyPoolStrategyBalanced(t *testing.T) { + opts := ApplyPoolStrategy(Options{}, PoolStrategy{ + Profile: PoolBalanced, + CPUCores: 4, + IOParallelism: 2, + }) + if opts.MaxOpenConns != 10 { + t.Fatalf("MaxOpenConns = %d", opts.MaxOpenConns) + } + if opts.MaxIdleConns != 3 { + t.Fatalf("MaxIdleConns = %d", opts.MaxIdleConns) + } +} + +func TestApplyPoolStrategyIOHeavyUsesExtraCapacity(t *testing.T) { + opts := ApplyPoolStrategy(Options{}, PoolStrategy{ + Profile: PoolIOHeavy, + CPUCores: 4, + IOParallelism: 4, + }) + if opts.MaxOpenConns != 24 { + t.Fatalf("MaxOpenConns = %d", opts.MaxOpenConns) + } + if opts.MaxIdleConns != 12 { + t.Fatalf("MaxIdleConns = %d", opts.MaxIdleConns) + } +} + +func TestApplyPoolStrategyLimit(t *testing.T) { + opts := ApplyPoolStrategy(Options{}, PoolStrategy{ + Profile: PoolIOHeavy, + CPUCores: 32, + IOParallelism: 32, + MaxOpenLimit: 20, + }) + if opts.MaxOpenConns != 20 { + t.Fatalf("MaxOpenConns = %d", opts.MaxOpenConns) + } + if opts.MaxIdleConns != 10 { + t.Fatalf("MaxIdleConns = %d", opts.MaxIdleConns) + } +} + +func TestFromEnvPoolProfile(t *testing.T) { + t.Setenv("DATABASE_POOL_PROFILE", "io-heavy") + t.Setenv("DATABASE_POOL_CPU_CORES", "4") + t.Setenv("DATABASE_IO_PARALLELISM", "4") + t.Setenv("DATABASE_POOL_MAX_OPEN_LIMIT", "30") + + opts := FromEnv() + if opts.MaxOpenConns != 24 || opts.MaxIdleConns != 12 { + t.Fatalf("pool profile not applied: %#v", opts) + } +} + +func TestFromEnvPoolKnobsWithoutProfile(t *testing.T) { + t.Setenv("DATABASE_POOL_CPU_CORES", "4") + t.Setenv("DATABASE_IO_PARALLELISM", "3") + t.Setenv("DATABASE_POOL_MAX_OPEN_LIMIT", "20") + + opts := FromEnv() + if opts.MaxOpenConns != 11 || opts.MaxIdleConns != 3 { + t.Fatalf("pool knobs not applied: %#v", opts) + } +} + +func TestFromEnvExplicitPoolOverridesProfile(t *testing.T) { + t.Setenv("DATABASE_POOL_PROFILE", "io-heavy") + t.Setenv("DATABASE_POOL_CPU_CORES", "4") + t.Setenv("DATABASE_IO_PARALLELISM", "4") + t.Setenv("DATABASE_MAX_OPEN_CONNS", "9") + t.Setenv("DATABASE_MAX_IDLE_CONNS", "2") + + opts := FromEnv() + if opts.MaxOpenConns != 9 || opts.MaxIdleConns != 2 { + t.Fatalf("explicit pool values should win: %#v", opts) + } +} diff --git a/registry/database/src/go/query.go b/registry/database/src/go/query.go new file mode 100644 index 0000000..308aa33 --- /dev/null +++ b/registry/database/src/go/query.go @@ -0,0 +1,251 @@ +package database + +import ( + "errors" + "fmt" + "sort" + "strconv" + "strings" +) + +const ( + maxFieldLen = 128 + maxValueLen = 1024 + maxConditions = 32 +) + +// ColumnMap maps user-facing filter/sort keys to trusted SQL column names. +// Column names must be simple identifiers or dot-qualified identifiers such as +// "users.created_at". +type ColumnMap map[string]string + +// Placeholder returns a placeholder for the 1-based argument position. +type Placeholder func(position int) string + +// QuestionPlaceholder returns "?" for MySQL, SQLite, and other drivers that +// use anonymous placeholders. +func QuestionPlaceholder(_ int) string { + return "?" +} + +// DollarPlaceholder returns PostgreSQL-style placeholders: $1, $2, ... +func DollarPlaceholder(position int) string { + if position < 1 { + position = 1 + } + return "$" + strconv.Itoa(position) +} + +// OrderBy converts a client sort key such as "created_at" or "-created_at" +// into a safe ORDER BY fragment using a whitelist. Empty input returns an empty +// fragment and nil error. +func OrderBy(raw string, columns ColumnMap) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", nil + } + if err := validateString("sort field", raw, maxFieldLen); err != nil { + return "", err + } + + desc := false + key := raw + if strings.HasPrefix(key, "-") { + desc = true + key = strings.TrimPrefix(key, "-") + } + column, err := lookupColumn(key, columns) + if err != nil { + return "", err + } + direction := "ASC" + if desc { + direction = "DESC" + } + return "ORDER BY " + column + " " + direction, nil +} + +// WhereEqual builds a deterministic WHERE fragment joined by AND. Filter keys +// are mapped through the whitelist; filter values are returned as args and are +// never concatenated into SQL. +func WhereEqual(filters map[string]string, columns ColumnMap, placeholder Placeholder) (string, []any, error) { + if len(filters) == 0 { + return "", nil, nil + } + if len(filters) > maxConditions { + return "", nil, errors.New("database: too many filter conditions") + } + b := NewBuilder(columns, placeholder) + keys := make([]string, 0, len(filters)) + for key := range filters { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if err := b.WhereEqual(key, filters[key]); err != nil { + return "", nil, err + } + } + return b.WhereSQL() +} + +// Builder incrementally constructs safe WHERE and ORDER BY fragments. +type Builder struct { + columns ColumnMap + placeholder Placeholder + where []string + args []any + order string +} + +// NewBuilder creates a query fragment builder. A nil placeholder defaults to +// QuestionPlaceholder. +func NewBuilder(columns ColumnMap, placeholder Placeholder) *Builder { + if placeholder == nil { + placeholder = QuestionPlaceholder + } + return &Builder{ + columns: columns, + placeholder: placeholder, + } +} + +// WhereEqual adds "column = placeholder" for a whitelisted field. +func (b *Builder) WhereEqual(field string, value any) error { + if len(b.where) >= maxConditions { + return errors.New("database: too many filter conditions") + } + column, err := lookupColumn(field, b.columns) + if err != nil { + return err + } + if err := validateArgValue(value); err != nil { + return err + } + b.args = append(b.args, value) + b.where = append(b.where, column+" = "+b.placeholder(len(b.args))) + return nil +} + +// OrderBy sets the ORDER BY fragment from a client sort key. +func (b *Builder) OrderBy(raw string) error { + order, err := OrderBy(raw, b.columns) + if err != nil { + return err + } + b.order = order + return nil +} + +// WhereSQL returns the WHERE fragment and a copy of its args. +func (b *Builder) WhereSQL() (string, []any, error) { + if len(b.where) == 0 { + return "", nil, nil + } + args := append([]any(nil), b.args...) + return "WHERE " + strings.Join(b.where, " AND "), args, nil +} + +// OrderSQL returns the ORDER BY fragment or an empty string. +func (b *Builder) OrderSQL() string { + return b.order +} + +// Reset clears all accumulated query fragments and arguments. It also drops +// references to old argument values so a reused Builder does not keep request +// data alive longer than necessary. +func (b *Builder) Reset() { + b.where = nil + b.args = nil + b.order = "" +} + +// SQL returns the combined WHERE and ORDER BY fragments plus WHERE args. +func (b *Builder) SQL() (string, []any, error) { + where, args, err := b.WhereSQL() + if err != nil { + return "", nil, err + } + switch { + case where == "": + return b.order, args, nil + case b.order == "": + return where, args, nil + default: + return where + " " + b.order, args, nil + } +} + +func lookupColumn(field string, columns ColumnMap) (string, error) { + field = strings.TrimSpace(field) + if err := validateRequiredString("field", field, maxFieldLen); err != nil { + return "", err + } + if columns == nil { + return "", errors.New("database: column whitelist is required") + } + column, ok := columns[field] + if !ok { + return "", fmt.Errorf("database: field is not allowed: %s", field) + } + if err := validateColumn(column); err != nil { + return "", err + } + return column, nil +} + +func validateColumn(column string) error { + if err := validateRequiredString("column", column, maxFieldLen); err != nil { + return err + } + parts := strings.Split(column, ".") + if len(parts) > 3 { + return errors.New("database: column whitelist contains too many identifier segments") + } + for _, part := range parts { + if !isIdentifier(part) { + return errors.New("database: column whitelist contains unsafe identifier") + } + } + return nil +} + +func validateArgValue(value any) error { + switch v := value.(type) { + case string: + return validateString("filter value", v, maxValueLen) + case []byte: + if len(v) > maxValueLen { + return errors.New("database: filter value is too long") + } + } + return nil +} + +func isIdentifier(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c == '_': + continue + case i == 0 && isASCIILetter(c): + continue + case i > 0 && (isASCIILetter(c) || isASCIIDigit(c)): + continue + default: + return false + } + } + return true +} + +func isASCIILetter(c byte) bool { + return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') +} + +func isASCIIDigit(c byte) bool { + return '0' <= c && c <= '9' +} diff --git a/registry/database/src/go/query_test.go b/registry/database/src/go/query_test.go new file mode 100644 index 0000000..bd252bd --- /dev/null +++ b/registry/database/src/go/query_test.go @@ -0,0 +1,220 @@ +package database + +import ( + "bytes" + "fmt" + "reflect" + "strings" + "testing" +) + +func testColumns() ColumnMap { + return ColumnMap{ + "id": "users.id", + "email": "users.email", + "created_at": "users.created_at", + "status": "users.status", + } +} + +func TestPlaceholders(t *testing.T) { + if QuestionPlaceholder(12) != "?" { + t.Fatal("question placeholder should ignore position") + } + if DollarPlaceholder(2) != "$2" { + t.Fatal("dollar placeholder mismatch") + } + if DollarPlaceholder(0) != "$1" { + t.Fatal("dollar placeholder should clamp to $1") + } +} + +func TestOrderBy(t *testing.T) { + tests := []struct { + raw string + want string + }{ + {"", ""}, + {"created_at", "ORDER BY users.created_at ASC"}, + {"-created_at", "ORDER BY users.created_at DESC"}, + {" email ", "ORDER BY users.email ASC"}, + } + for _, tt := range tests { + t.Run(tt.raw, func(t *testing.T) { + got, err := OrderBy(tt.raw, testColumns()) + if err != nil { + t.Fatalf("OrderBy: %v", err) + } + if got != tt.want { + t.Fatalf("got %q want %q", got, tt.want) + } + }) + } +} + +func TestOrderByRejectsUnsafeInput(t *testing.T) { + tests := []string{ + "created_at DESC", + "created_at;DROP TABLE users", + "missing", + "bad\r\nfield", + strings.Repeat("a", maxFieldLen+1), + } + for _, raw := range tests { + t.Run(raw, func(t *testing.T) { + if _, err := OrderBy(raw, testColumns()); err == nil { + t.Fatal("expected error") + } + }) + } +} + +func TestWhereEqual(t *testing.T) { + sql, args, err := WhereEqual(map[string]string{ + "status": "active", + "email": "ada@example.com", + }, testColumns(), DollarPlaceholder) + if err != nil { + t.Fatalf("WhereEqual: %v", err) + } + if sql != "WHERE users.email = $1 AND users.status = $2" { + t.Fatalf("sql = %q", sql) + } + if !reflect.DeepEqual(args, []any{"ada@example.com", "active"}) { + t.Fatalf("args = %#v", args) + } +} + +func TestBuilder(t *testing.T) { + b := NewBuilder(testColumns(), QuestionPlaceholder) + if err := b.WhereEqual("email", "ada@example.com"); err != nil { + t.Fatalf("WhereEqual email: %v", err) + } + if err := b.WhereEqual("status", "active"); err != nil { + t.Fatalf("WhereEqual status: %v", err) + } + if err := b.OrderBy("-created_at"); err != nil { + t.Fatalf("OrderBy: %v", err) + } + sql, args, err := b.SQL() + if err != nil { + t.Fatalf("SQL: %v", err) + } + if sql != "WHERE users.email = ? AND users.status = ? ORDER BY users.created_at DESC" { + t.Fatalf("sql = %q", sql) + } + if !reflect.DeepEqual(args, []any{"ada@example.com", "active"}) { + t.Fatalf("args = %#v", args) + } +} + +func TestBuilderDefaultsPlaceholder(t *testing.T) { + b := NewBuilder(testColumns(), nil) + if err := b.WhereEqual("id", 123); err != nil { + t.Fatalf("WhereEqual: %v", err) + } + sql, args, err := b.WhereSQL() + if err != nil { + t.Fatalf("WhereSQL: %v", err) + } + if sql != "WHERE users.id = ?" { + t.Fatalf("sql = %q", sql) + } + if !reflect.DeepEqual(args, []any{123}) { + t.Fatalf("args = %#v", args) + } +} + +func TestBuilderReset(t *testing.T) { + b := NewBuilder(testColumns(), DollarPlaceholder) + if err := b.WhereEqual("email", "ada@example.com"); err != nil { + t.Fatalf("WhereEqual: %v", err) + } + if err := b.OrderBy("-created_at"); err != nil { + t.Fatalf("OrderBy: %v", err) + } + b.Reset() + sql, args, err := b.SQL() + if err != nil { + t.Fatalf("SQL after reset: %v", err) + } + if sql != "" || args != nil { + t.Fatalf("reset SQL = %q args=%#v", sql, args) + } + if err := b.WhereEqual("status", "active"); err != nil { + t.Fatalf("WhereEqual after reset: %v", err) + } + sql, args, err = b.WhereSQL() + if err != nil { + t.Fatalf("WhereSQL after reset: %v", err) + } + if sql != "WHERE users.status = $1" { + t.Fatalf("sql = %q", sql) + } + if !reflect.DeepEqual(args, []any{"active"}) { + t.Fatalf("args = %#v", args) + } +} + +func TestWhereEqualRejectsTooManyFilters(t *testing.T) { + filters := make(map[string]string, maxConditions+1) + for i := 0; i < maxConditions+1; i++ { + filters[fmt.Sprintf("f%d", i)] = "x" + } + if _, _, err := WhereEqual(filters, testColumns(), QuestionPlaceholder); err == nil { + t.Fatal("expected too many filters error") + } +} + +func TestBuilderRejectsTooManyConditions(t *testing.T) { + columns := make(ColumnMap, maxConditions+1) + for i := 0; i < maxConditions+1; i++ { + name := fmt.Sprintf("f%d", i) + columns[name] = "users." + name + } + b := NewBuilder(columns, QuestionPlaceholder) + for i := 0; i < maxConditions; i++ { + name := fmt.Sprintf("f%d", i) + if err := b.WhereEqual(name, "x"); err != nil { + t.Fatalf("WhereEqual %d: %v", i, err) + } + } + if err := b.WhereEqual(fmt.Sprintf("f%d", maxConditions), "x"); err == nil { + t.Fatal("expected too many conditions error") + } +} + +func TestUnsafeWhitelistColumnRejected(t *testing.T) { + columns := ColumnMap{"name": "users.name; DROP TABLE users"} + if _, err := OrderBy("name", columns); err == nil { + t.Fatal("expected unsafe whitelist error") + } +} + +func TestColumnWhitelistRejectsNonPortableIdentifiers(t *testing.T) { + tests := []ColumnMap{ + {"name": "users.名字"}, + {"name": "app.public.users.name"}, + } + for _, columns := range tests { + if _, err := OrderBy("name", columns); err == nil { + t.Fatalf("expected unsafe whitelist error for %#v", columns) + } + } +} + +func TestWhereEqualRejectsUnsafeValue(t *testing.T) { + if _, _, err := WhereEqual(map[string]string{"email": "a\r\nb"}, testColumns(), QuestionPlaceholder); err == nil { + t.Fatal("expected CRLF value error") + } + if _, _, err := WhereEqual(map[string]string{"email": strings.Repeat("a", maxValueLen+1)}, testColumns(), QuestionPlaceholder); err == nil { + t.Fatal("expected long value error") + } +} + +func TestBuilderRejectsOversizedByteValue(t *testing.T) { + b := NewBuilder(testColumns(), QuestionPlaceholder) + if err := b.WhereEqual("email", bytes.Repeat([]byte("a"), maxValueLen+1)); err == nil { + t.Fatal("expected long byte value error") + } +} diff --git a/registry/database/src/go/transaction.go b/registry/database/src/go/transaction.go new file mode 100644 index 0000000..615bebf --- /dev/null +++ b/registry/database/src/go/transaction.go @@ -0,0 +1,62 @@ +package database + +import ( + "context" + "database/sql" + "errors" + "fmt" +) + +// DBTX is the common subset implemented by *sql.DB and *sql.Tx. Repository +// methods can accept this interface to run either inside or outside a +// transaction. +type DBTX interface { + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row + PrepareContext(ctx context.Context, query string) (*sql.Stmt, error) +} + +// Beginner is implemented by *sql.DB and allows tests or adapters to provide a +// transaction-capable database handle. +type Beginner interface { + BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) +} + +// WithinTx runs fn inside a transaction. It commits when fn returns nil, +// rolls back when fn returns an error, and also rolls back before re-panicking +// if fn panics. +func WithinTx(ctx context.Context, db Beginner, opts *sql.TxOptions, fn func(context.Context, *sql.Tx) error) (err error) { + if ctx == nil { + return errors.New("database: context is nil") + } + if db == nil { + return errors.New("database: transaction beginner is nil") + } + if fn == nil { + return errors.New("database: transaction function is nil") + } + + tx, err := db.BeginTx(ctx, opts) + if err != nil { + return errors.New("database: begin transaction failed") + } + + defer func() { + if recovered := recover(); recovered != nil { + _ = tx.Rollback() + panic(recovered) + } + }() + + if err := fn(ctx, tx); err != nil { + if rbErr := tx.Rollback(); rbErr != nil { + return fmt.Errorf("database: transaction failed: %w; rollback failed: %v", err, rbErr) + } + return err + } + if err := tx.Commit(); err != nil { + return errors.New("database: commit transaction failed") + } + return nil +} diff --git a/registry/database/src/go/transaction_test.go b/registry/database/src/go/transaction_test.go new file mode 100644 index 0000000..c3d27e6 --- /dev/null +++ b/registry/database/src/go/transaction_test.go @@ -0,0 +1,92 @@ +package database + +import ( + "context" + "database/sql" + "errors" + "testing" +) + +func openTxTestDB(t *testing.T, dsn string) *sql.DB { + t.Helper() + ensureTestDriver() + db, err := sql.Open(testDriverName, dsn) + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return db +} + +func TestWithinTxCommitsOnNilError(t *testing.T) { + resetTxCounters() + db := openTxTestDB(t, "memory") + err := WithinTx(context.Background(), db, nil, func(ctx context.Context, tx *sql.Tx) error { + if tx == nil { + t.Fatal("tx is nil") + } + return nil + }) + if err != nil { + t.Fatalf("WithinTx: %v", err) + } + if testTxCounters.commits.Load() != 1 || testTxCounters.rollbacks.Load() != 0 { + t.Fatalf("unexpected counters commits=%d rollbacks=%d", testTxCounters.commits.Load(), testTxCounters.rollbacks.Load()) + } +} + +func TestWithinTxRollsBackOnError(t *testing.T) { + resetTxCounters() + db := openTxTestDB(t, "memory") + want := errors.New("domain error") + err := WithinTx(context.Background(), db, nil, func(context.Context, *sql.Tx) error { + return want + }) + if !errors.Is(err, want) { + t.Fatalf("expected domain error, got %v", err) + } + if testTxCounters.commits.Load() != 0 || testTxCounters.rollbacks.Load() != 1 { + t.Fatalf("unexpected counters commits=%d rollbacks=%d", testTxCounters.commits.Load(), testTxCounters.rollbacks.Load()) + } +} + +func TestWithinTxRollsBackOnPanic(t *testing.T) { + resetTxCounters() + db := openTxTestDB(t, "memory") + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic") + } + if testTxCounters.commits.Load() != 0 || testTxCounters.rollbacks.Load() != 1 { + t.Fatalf("unexpected counters commits=%d rollbacks=%d", testTxCounters.commits.Load(), testTxCounters.rollbacks.Load()) + } + }() + _ = WithinTx(context.Background(), db, nil, func(context.Context, *sql.Tx) error { + panic("boom") + }) +} + +func TestWithinTxRejectsNilInputs(t *testing.T) { + db := openTxTestDB(t, "memory") + if err := WithinTx(nil, db, nil, func(context.Context, *sql.Tx) error { return nil }); err == nil { + t.Fatal("expected nil context error") + } + if err := WithinTx(context.Background(), nil, nil, func(context.Context, *sql.Tx) error { return nil }); err == nil { + t.Fatal("expected nil db error") + } + if err := WithinTx(context.Background(), db, nil, nil); err == nil { + t.Fatal("expected nil function error") + } +} + +func TestWithinTxBeginAndCommitErrorsAreSanitized(t *testing.T) { + db := openTxTestDB(t, "begin-error") + if err := WithinTx(context.Background(), db, nil, func(context.Context, *sql.Tx) error { return nil }); err == nil { + t.Fatal("expected begin error") + } + + db = openTxTestDB(t, "commit-error") + if err := WithinTx(context.Background(), db, nil, func(context.Context, *sql.Tx) error { return nil }); err == nil { + t.Fatal("expected commit error") + } +} diff --git a/registry/file-upload/src/go/storage.go b/registry/file-upload/src/go/storage.go index c19ac45..d9345b8 100644 --- a/registry/file-upload/src/go/storage.go +++ b/registry/file-upload/src/go/storage.go @@ -18,6 +18,8 @@ var ( ErrInvalidName = errors.New("fileupload: invalid filename") ) +const defaultMemoryStorageMaxFiles = 1024 + // Storage abstracts where and how uploaded files are persisted. Implementations // must reject any name that contains path separators or traversal segments so // that path-traversal attacks cannot escape the storage root. @@ -146,15 +148,18 @@ func (s *LocalStorage) Exists(ctx context.Context, name string) bool { // services, and deployments where persistence is handled elsewhere. type MemoryStorage struct { URLPrefix string + MaxFiles int - mu sync.RWMutex + mu sync.Mutex files map[string][]byte + order []string } // NewMemoryStorage creates an empty MemoryStorage. func NewMemoryStorage(urlPrefix string) *MemoryStorage { return &MemoryStorage{ URLPrefix: urlPrefix, + MaxFiles: defaultMemoryStorageMaxFiles, files: make(map[string][]byte), } } @@ -167,36 +172,106 @@ func (s *MemoryStorage) Save(ctx context.Context, name string, data []byte) (str cp := make([]byte, len(data)) copy(cp, data) s.mu.Lock() + defer s.mu.Unlock() + if s.files == nil { + s.files = make(map[string][]byte) + } + if _, exists := s.files[name]; exists { + s.files[name] = cp + s.touchLocked(name) + return strings.TrimRight(s.URLPrefix, "/") + "/" + name, nil + } + for len(s.files) >= s.maxFilesLocked() { + s.evictOldestLocked() + } s.files[name] = cp - s.mu.Unlock() + s.order = append(s.order, name) return strings.TrimRight(s.URLPrefix, "/") + "/" + name, nil } // Get implements Storage. func (s *MemoryStorage) Get(ctx context.Context, name string) ([]byte, error) { - s.mu.RLock() + if err := safeName(name); err != nil { + return nil, err + } + s.mu.Lock() data, ok := s.files[name] - s.mu.RUnlock() if !ok { + s.mu.Unlock() return nil, ErrFileNotFound } + s.touchLocked(name) out := make([]byte, len(data)) copy(out, data) + s.mu.Unlock() return out, nil } // Delete implements Storage. func (s *MemoryStorage) Delete(ctx context.Context, name string) error { + if err := safeName(name); err != nil { + return err + } s.mu.Lock() delete(s.files, name) + s.removeFromOrderLocked(name) s.mu.Unlock() return nil } // Exists implements Storage. func (s *MemoryStorage) Exists(ctx context.Context, name string) bool { - s.mu.RLock() + if err := safeName(name); err != nil { + return false + } + s.mu.Lock() _, ok := s.files[name] - s.mu.RUnlock() + if ok { + s.touchLocked(name) + } + s.mu.Unlock() return ok } + +func (s *MemoryStorage) maxFilesLocked() int { + if s.MaxFiles <= 0 { + return defaultMemoryStorageMaxFiles + } + return s.MaxFiles +} + +func (s *MemoryStorage) evictOldestLocked() { + for len(s.order) > 0 { + name := s.order[0] + s.order = s.order[1:] + if _, ok := s.files[name]; ok { + delete(s.files, name) + return + } + } + for name := range s.files { + delete(s.files, name) + return + } +} + +func (s *MemoryStorage) touchLocked(name string) { + for i, existing := range s.order { + if existing == name { + copy(s.order[i:], s.order[i+1:]) + s.order[len(s.order)-1] = name + return + } + } + s.order = append(s.order, name) +} + +func (s *MemoryStorage) removeFromOrderLocked(name string) { + for i, existing := range s.order { + if existing == name { + copy(s.order[i:], s.order[i+1:]) + s.order = s.order[:len(s.order)-1] + return + } + } +} diff --git a/registry/file-upload/src/go/storage_test.go b/registry/file-upload/src/go/storage_test.go index e561137..ef0b17a 100644 --- a/registry/file-upload/src/go/storage_test.go +++ b/registry/file-upload/src/go/storage_test.go @@ -49,3 +49,35 @@ func TestStorageMemoryCopiesData(t *testing.T) { t.Fatal("file should not exist after delete") } } + +func TestMemoryStorageEvictsLeastRecentlyUsed(t *testing.T) { + ctx := context.Background() + storage := NewMemoryStorage("/files") + storage.MaxFiles = 2 + + if _, err := storage.Save(ctx, "a.txt", []byte("a")); err != nil { + t.Fatalf("Save a: %v", err) + } + if _, err := storage.Save(ctx, "b.txt", []byte("b")); err != nil { + t.Fatalf("Save b: %v", err) + } + if _, err := storage.Get(ctx, "a.txt"); err != nil { + t.Fatalf("Get a: %v", err) + } + if _, err := storage.Save(ctx, "c.txt", []byte("c")); err != nil { + t.Fatalf("Save c: %v", err) + } + + if !storage.Exists(ctx, "a.txt") { + t.Fatal("recently used file should still exist") + } + if storage.Exists(ctx, "b.txt") { + t.Fatal("least recently used file should be evicted") + } + if !storage.Exists(ctx, "c.txt") { + t.Fatal("new file should exist") + } + if _, err := storage.Get(ctx, "b.txt"); !errors.Is(err, ErrFileNotFound) { + t.Fatalf("Get evicted file = %v, want ErrFileNotFound", err) + } +} diff --git a/registry/index.json b/registry/index.json index 2905080..193e6e0 100644 --- a/registry/index.json +++ b/registry/index.json @@ -1,7 +1,7 @@ { "schemaVersion": 1, "name": "Scion", - "version": "0.1.0", + "version": "0.2.0", "description": "Backend pattern registry", "patterns": [ { @@ -38,6 +38,23 @@ "standaloneInclude": ["go.mod", "go.sum"], "status": "ready" }, + { + "id": "database", + "name": "Database Helpers", + "description": "database/sql connection setup, transaction helper, and safe query fragments", + "path": "registry/database", + "languages": ["go"], + "frameworks": [], + "tags": ["database", "sql", "transactions", "connection-pool", "security"], + "version": "0.1.0", + "package": "database", + "source": "registry/database/src/go", + "defaultTarget": "internal/database", + "stdlibOnly": true, + "include": ["*.go"], + "standaloneInclude": ["go.mod", "go.sum"], + "status": "ready" + }, { "id": "middleware", "name": "HTTP Middleware", diff --git a/registry/middleware/src/go/config.go b/registry/middleware/src/go/config.go index 69ee73a..1d8515d 100644 --- a/registry/middleware/src/go/config.go +++ b/registry/middleware/src/go/config.go @@ -114,7 +114,8 @@ type ProxyOptions struct { TrustedProxies []string // ProxyCount is the number of trusted proxy layers. 0 = trust none. - // If both set, TrustedProxies takes precedence. + // Forwarded headers are considered only when r.RemoteAddr is inside + // TrustedProxies; ProxyCount alone never makes arbitrary clients trusted. ProxyCount int } diff --git a/registry/middleware/src/go/proxy.go b/registry/middleware/src/go/proxy.go index 840365b..740b432 100644 --- a/registry/middleware/src/go/proxy.go +++ b/registry/middleware/src/go/proxy.go @@ -8,14 +8,12 @@ import ( ) const maxProxyCount = 10 +const maxProxyHeaderLen = 1024 // ClientIP extracts the client IP address from the request. -// It first checks the context (set by TrustedProxy middleware). -// If not found, it falls back to RemoteAddr. +// SECURITY: it always returns r.RemoteAddr (with any port stripped). It does +// not trust X-Forwarded-For, X-Real-IP, or values stored in request context. func ClientIP(r *http.Request) string { - if ip, ok := r.Context().Value(clientIPKey).(string); ok && ip != "" { - return ip - } return remoteAddrIP(r) } @@ -84,6 +82,10 @@ func clientIPFromConfig(r *http.Request, cfg proxyConfig) string { if cfg.proxyCount == 0 && len(cfg.nets) == 0 { return remoteAddrIP(r) } + remoteIP := remoteAddrIP(r) + if len(cfg.nets) == 0 || !isTrustedNets(remoteIP, cfg.nets) { + return remoteAddrIP(r) + } // Try X-Forwarded-For first. xff := r.Header.Get("X-Forwarded-For") @@ -94,9 +96,12 @@ func clientIPFromConfig(r *http.Request, cfg proxyConfig) string { if xff == "" { return remoteAddrIP(r) } + if len(xff) > maxProxyHeaderLen || strings.ContainsAny(xff, "\r\n\x00") { + return remoteAddrIP(r) + } // X-Forwarded-For: client, proxy1, proxy2, ..., proxyN - ips := strings.Split(xff, ",") + ips := strings.SplitN(xff, ",", maxProxyCount+2) // ProxyCount mode. if cfg.proxyCount > 0 { diff --git a/registry/middleware/src/go/proxy_test.go b/registry/middleware/src/go/proxy_test.go index a579978..edb082a 100644 --- a/registry/middleware/src/go/proxy_test.go +++ b/registry/middleware/src/go/proxy_test.go @@ -1,6 +1,7 @@ package middleware import ( + "context" "net" "net/http" "net/http/httptest" @@ -20,9 +21,10 @@ func TestClientIPRemoteAddr(t *testing.T) { func TestClientIPProxyCount(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/", nil) + req.RemoteAddr = "10.0.0.1:12345" req.Header.Set("X-Forwarded-For", " 1.1.1.1, 2.2.2.2, 3.3.3.3 ") - opts := ProxyOptions{ProxyCount: 2} + opts := ProxyOptions{TrustedProxies: []string{"10.0.0.0/8"}, ProxyCount: 2} ip := ClientIPWithOptions(req, opts) if ip != "1.1.1.1" { @@ -32,6 +34,7 @@ func TestClientIPProxyCount(t *testing.T) { func TestClientIPCIDRWhitelist(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/", nil) + req.RemoteAddr = "10.0.0.1:12345" req.Header.Set("X-Forwarded-For", "1.1.1.1, 10.0.0.1") opts := ProxyOptions{ @@ -60,10 +63,11 @@ func TestClientIPNoXFF(t *testing.T) { func TestClientIPRealIP(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/", nil) + req.RemoteAddr = "10.0.0.1:12345" req.Header.Set("X-Real-IP", "203.0.113.50") // No X-Forwarded-For, so X-Real-IP is used as fallback. - opts := ProxyOptions{ProxyCount: 1} + opts := ProxyOptions{TrustedProxies: []string{"10.0.0.0/8"}, ProxyCount: 1} ip := ClientIPWithOptions(req, opts) if ip != "203.0.113.50" { @@ -84,6 +88,7 @@ func TestTrustedProxyMiddleware(t *testing.T) { handler := mw(next) req := httptest.NewRequest(http.MethodGet, "/", nil) + req.RemoteAddr = "10.0.0.1:12345" req.Header.Set("X-Forwarded-For", "5.5.5.5, 10.0.0.1") rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) @@ -95,10 +100,11 @@ func TestTrustedProxyMiddleware(t *testing.T) { func TestProxyCountClamp(t *testing.T) { req := httptest.NewRequest(http.MethodGet, "/", nil) + req.RemoteAddr = "10.0.0.1:12345" req.Header.Set("X-Forwarded-For", "1.1.1.1, 2.2.2.2, 3.3.3.3, 4.4.4.4, 5.5.5.5") // ProxyCount=100 should be clamped to 10 (maxProxyCount). - opts := ProxyOptions{ProxyCount: 100} + opts := ProxyOptions{TrustedProxies: []string{"10.0.0.0/8"}, ProxyCount: 100} ip := ClientIPWithOptions(req, opts) // With 5 IPs and ProxyCount=10 (clamped): idx = 5 - 10 - 1 = -6 < 0, @@ -108,6 +114,28 @@ func TestProxyCountClamp(t *testing.T) { } } +func TestClientIPIgnoresTrustedProxyContext(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.RemoteAddr = "5.6.7.8:12345" + ctx := context.WithValue(req.Context(), clientIPKey, "1.1.1.1") + req = req.WithContext(ctx) + + if got := ClientIP(req); got != "5.6.7.8" { + t.Fatalf("ClientIP = %q, want RemoteAddr", got) + } +} + +func TestClientIPWithOptionsRejectsUntrustedRemote(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.RemoteAddr = "5.6.7.8:12345" + req.Header.Set("X-Forwarded-For", "1.1.1.1") + + ip := ClientIPWithOptions(req, ProxyOptions{TrustedProxies: []string{"10.0.0.0/8"}}) + if ip != "5.6.7.8" { + t.Fatalf("ClientIPWithOptions = %q, want RemoteAddr", ip) + } +} + func TestIsTrustedProxy(t *testing.T) { tests := []struct { ip string diff --git a/registry/rbac/src/go/manager.go b/registry/rbac/src/go/manager.go index 5cdfee3..93874d8 100644 --- a/registry/rbac/src/go/manager.go +++ b/registry/rbac/src/go/manager.go @@ -17,6 +17,17 @@ var ErrCircularDependency = errors.New("circular dependency detected in role hie // ErrInvalidName is returned when a name fails validation. var ErrInvalidName = errors.New("invalid name: must be non-empty, <= 128 chars, no CRLF or null bytes") +// ErrLimitExceeded is returned when an in-memory collection reaches its +// safety cap. +var ErrLimitExceeded = errors.New("capacity limit exceeded") + +const ( + maxRoles = 10000 + maxUsers = 10000 + maxRoleParents = 64 + maxRolePermissions = 256 +) + // Manager manages roles, users, and permission checks. // All methods are safe for concurrent use. type Manager struct { @@ -39,6 +50,9 @@ func (m *Manager) AddRole(role *Role) error { if role == nil || !validateName(role.Name) { return ErrInvalidName } + if len(role.Permissions) > maxRolePermissions || len(role.Parents) > maxRoleParents { + return ErrLimitExceeded + } for _, p := range role.Permissions { if !validateName(p.Resource) || !validateName(p.Action) { return ErrInvalidName @@ -56,6 +70,9 @@ func (m *Manager) AddRole(role *Role) error { if _, exists := m.roles[role.Name]; exists { return ErrRoleExists } + if len(m.roles) >= maxRoles { + return ErrLimitExceeded + } // Check for circular dependencies before adding. for _, parent := range role.Parents { @@ -150,6 +167,9 @@ func (m *Manager) AssignRole(userID, roleName string) error { user, exists := m.users[userID] if !exists { + if len(m.users) >= maxUsers { + return ErrLimitExceeded + } user = &User{ID: userID} m.users[userID] = user } diff --git a/registry/rbac/src/go/manager_test.go b/registry/rbac/src/go/manager_test.go index 1210ecc..dd17d61 100644 --- a/registry/rbac/src/go/manager_test.go +++ b/registry/rbac/src/go/manager_test.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "net/http/httptest" + "strconv" "strings" "sync" "testing" @@ -243,3 +244,48 @@ func TestConcurrentAccess(t *testing.T) { } wg.Wait() } + +func TestManagerCapacityLimits(t *testing.T) { + m := NewManager() + for i := 0; i < maxRoles; i++ { + name := "role" + strconv.Itoa(i) + if err := m.AddRole(&Role{Name: name}); err != nil { + t.Fatalf("AddRole %d: %v", i, err) + } + } + if err := m.AddRole(&Role{Name: "overflow"}); err != ErrLimitExceeded { + t.Fatalf("expected ErrLimitExceeded for role overflow, got %v", err) + } + + users := NewManager() + if err := users.AddRole(&Role{Name: "base"}); err != nil { + t.Fatalf("AddRole base: %v", err) + } + for i := 0; i < maxUsers; i++ { + if err := users.AssignRole("user"+strconv.Itoa(i), "base"); err != nil { + t.Fatalf("AssignRole %d: %v", i, err) + } + } + if err := users.AssignRole("overflow", "base"); err != ErrLimitExceeded { + t.Fatalf("expected ErrLimitExceeded for user overflow, got %v", err) + } +} + +func TestRoleSliceLimits(t *testing.T) { + m := NewManager() + perms := make([]Permission, maxRolePermissions+1) + for i := range perms { + perms[i] = Permission{Resource: "r", Action: "a"} + } + if err := m.AddRole(&Role{Name: "too-many-perms", Permissions: perms}); err != ErrLimitExceeded { + t.Fatalf("expected ErrLimitExceeded for permissions, got %v", err) + } + + parents := make([]string, maxRoleParents+1) + for i := range parents { + parents[i] = "parent" + strconv.Itoa(i) + } + if err := m.AddRole(&Role{Name: "too-many-parents", Parents: parents}); err != ErrLimitExceeded { + t.Fatalf("expected ErrLimitExceeded for parents, got %v", err) + } +}