diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eb9323c..1899b73 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,35 +1,36 @@ name: CI on: - push: - branches: [main] - pull_request: - branches: [main] + push: + branches: [main] + pull_request: + branches: [main] jobs: - verify-tomls: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Bun - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - - name: Install dependencies - run: bun install - working-directory: ci/bun - - - name: Collect snippets - run: bun run ci/bun/scripts/collectSnippets.ts - - - name: Verify snippets - run: bash ci/bash/verify_snippets.bash - - - name: Upload log artifact - if: always() - uses: actions/upload-artifact@v4 - with: - name: verification-log - path: verification-log.json + verify-tomls: + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Checkout another public repo + uses: actions/checkout@v4 + with: + repository: pgdogdev/pgdog + ref: main + path: pgdog-source + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - name: Install CMake 3.31 + run: | + sudo apt update && sudo apt install mold -y + sudo apt remove cmake + sudo pip3 install cmake==3.31.6 + cmake --version + - name: Build + run: cargo build + working-directory: pgdog-source + - name: Install test dependencies + run: sudo pip3 install -r tests/requirements.txt + - name: Run tests + run: python3 tests/test_code_blocks.py pgdog-source/target/debug/pgdog diff --git a/ci/bash/verify_snippets.bash b/ci/bash/verify_snippets.bash deleted file mode 100644 index 54ca4ef..0000000 --- a/ci/bash/verify_snippets.bash +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -set -euo pipefail - -OUTPUT_DIRECTORY="${PWD}/ci/tmp" - -# DOCKER_IMAGE = "ghcr.io/pgdogdev/pgdog:main"; <- change when cache if fixed -DOCKER_IMAGE="ghcr.io/pgdogdev/pgdog:main@sha256:3036d2ac7b684643dd187c42971f003f9d76e5f54cd129dcba742c309d7debd0" - -# nuke previous snippets -rm -rf "$OUTPUT_DIRECTORY" -mkdir -p "$OUTPUT_DIRECTORY" - -# Pull image once -docker pull "$DOCKER_IMAGE" - -# Run one container and do all checks inside it -docker run --rm \ - -v "$OUTPUT_DIRECTORY":/ci/tmp \ - "$DOCKER_IMAGE" \ - sh -euc ' - ANY_FAILED=0 - - for toml in /ci/tmp/*.toml; do - [ -f "$toml" ] || continue - - orig=$(head -n1 "$toml" | sed "s/# file: //") - line=$(head -n2 "$toml" | tail -n1 | sed "s/# line_number: //") - - if pgdog configcheck --config "$toml" >/dev/null 2>&1 \ - || pgdog configcheck --users "$toml" >/dev/null 2>&1; then - echo "${orig}:${line} verified successfully" - else - echo "Validation failed for ${orig}:${line}" >&2 - ANY_FAILED=1 - fi - done - - if [ "$ANY_FAILED" -eq 1 ]; then - echo "Some snippets failed verification!" >&2 - exit 1 - else - echo "All snippets verified successfully!" - exit 0 - fi - ' diff --git a/ci/bun/bun.lock b/ci/bun/bun.lock deleted file mode 100644 index 7b784bd..0000000 --- a/ci/bun/bun.lock +++ /dev/null @@ -1,29 +0,0 @@ -{ - "lockfileVersion": 1, - "workspaces": { - "": { - "name": "bun", - "devDependencies": { - "@types/bun": "latest" - }, - "peerDependencies": { - "typescript": "^5" - } - } - }, - "packages": { - "@types/bun": ["@types/bun@1.2.19", "", { "dependencies": { "bun-types": "1.2.19" } }, "sha512-d9ZCmrH3CJ2uYKXQIUuZ/pUnTqIvLDS0SK7pFmbx8ma+ziH/FRMoAq5bYpRG7y+w1gl+HgyNZbtqgMq4W4e2Lg=="], - - "@types/node": ["@types/node@24.1.0", "", { "dependencies": { "undici-types": "~7.8.0" } }, "sha512-ut5FthK5moxFKH2T1CUOC6ctR67rQRvvHdFLCD2Ql6KXmMuCrjsSsRI9UsLCm9M18BMwClv4pn327UvB7eeO1w=="], - - "@types/react": ["@types/react@19.1.9", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-WmdoynAX8Stew/36uTSVMcLJJ1KRh6L3IZRx1PZ7qJtBqT3dYTgyDTx8H1qoRghErydW7xw9mSJ3wS//tCRpFA=="], - - "bun-types": ["bun-types@1.2.19", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-uAOTaZSPuYsWIXRpj7o56Let0g/wjihKCkeRqUBhlLVM/Bt+Fj9xTo+LhC1OV1XDaGkz4hNC80et5xgy+9KTHQ=="], - - "csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="], - - "typescript": ["typescript@5.9.2", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A=="], - - "undici-types": ["undici-types@7.8.0", "", {}, "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw=="] - } -} diff --git a/ci/bun/package.json b/ci/bun/package.json deleted file mode 100644 index 5808d97..0000000 --- a/ci/bun/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "bun", - "module": "index.ts", - "type": "module", - "private": true, - "devDependencies": { - "@types/bun": "latest" - }, - "peerDependencies": { - "typescript": "^5" - } -} \ No newline at end of file diff --git a/ci/bun/scripts/collectSnippets.ts b/ci/bun/scripts/collectSnippets.ts deleted file mode 100644 index 4c83e17..0000000 --- a/ci/bun/scripts/collectSnippets.ts +++ /dev/null @@ -1,131 +0,0 @@ -// ----------------------------------------------------------------------------- -// ----- Constants ------------------------------------------------------------- - -const TOML_SNIPPET_REGEX = /^([ \t]*)```toml[^\n]*\r?\n([\s\S]*?)^\1```/gm; - -const DOCS_ROOT_DIRECTORY = `${process.cwd()}/docs`; -const OUTPUT_DIRECTORY = `${process.cwd()}/ci/tmp`; - -const SnippetsByMd5 = new Map(); - -// ----------------------------------------------------------------------------- -// ----- Invoke main() --------------------------------------------------------- - -main(); - -// ----------------------------------------------------------------------------- -// ----- Main ------------------------------------------------------------------ - -async function main() { - await setup(); - - // Scan docs for TOML snippets - const snippets = await extractTomlSnippets(); - - // Create one temporary {md5}.toml file for each snippet - await Promise.all(snippets.map(createFileForSnippet)); -} - -// ----------------------------------------------------------------------------- -// ----- Types ----------------------------------------------------------------- - -type Snippet = { - file: string; - content: string; - line: number; -}; - -// ----------------------------------------------------------------------------- -// ----- Helpers --------------------------------------------------------------- - -async function extractTomlSnippets() { - const glob = new Bun.Glob("**/*.md"); - const paths = glob.scan(DOCS_ROOT_DIRECTORY); - - const snippets: Snippet[] = []; - - for await (const relativePath of paths) { - const fullPath = `${DOCS_ROOT_DIRECTORY}/${relativePath}`; - const fileText = await Bun.file(fullPath).text(); - - while (true) { - const match = TOML_SNIPPET_REGEX.exec(fileText); - if (match === null) { - break; - } - - const codeIndent = match[1]; - const deindentedContent = match[2] - .split(/\r?\n/) - .map((line) => - line.startsWith(codeIndent) ? line.slice(codeIndent.length) : line, - ) - .join("\n") - .trim(); - - const startIndex = match.index; - const startLineNumber = fileText - .slice(0, startIndex) - .split(/\r?\n/).length; - - const snippet = { - file: relativePath, - content: deindentedContent, - line: startLineNumber, - }; - - const md5 = hash(snippet); - - SnippetsByMd5.set(md5, snippet); - snippets.push(snippet); - } - } - - return snippets; -} - -// ----------------------------------------------------------------------------- - -function hash(snippet: Snippet): string { - const hasher = new Bun.CryptoHasher("md5"); - hasher.update(JSON.stringify(snippet)); - const md5 = hasher.digest("hex"); - return md5; -} - -// ----------------------------------------------------------------------------- - -async function setup() { - await Bun.$`mkdir -p ${OUTPUT_DIRECTORY}`; - await Bun.$`rm -rf ${OUTPUT_DIRECTORY}`.nothrow(); -} - -// ----------------------------------------------------------------------------- - -async function createFileForSnippet(snippet: Snippet) { - const prefix = guessType(snippet.content); - const md5 = hash(snippet); - const filename = `${prefix}_${md5}.toml`; - const outputFilePath = `${OUTPUT_DIRECTORY}/${filename}`; - - const content = [ - `# file: ${snippet.file}`, - `# line_number: ${snippet.line}`, - "", - `${snippet.content}`, - "", - ].join("\n"); - - await Bun.write(outputFilePath, content); -} - -// ----------------------------------------------------------------------------- - -type ConfigOrUsers = "config" | "users"; - -function guessType(snippetContent: string): ConfigOrUsers { - return snippetContent.includes("[[users]]") ? "users" : "config"; -} - -// ----------------------------------------------------------------------------- -// ----------------------------------------------------------------------------- diff --git a/ci/bun/tsconfig.json b/ci/bun/tsconfig.json deleted file mode 100644 index 238655f..0000000 --- a/ci/bun/tsconfig.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "compilerOptions": { - // Enable latest features - "lib": ["ESNext", "DOM"], - "target": "ESNext", - "module": "ESNext", - "moduleDetection": "force", - "jsx": "react-jsx", - "allowJs": true, - - // Bundler mode - "moduleResolution": "bundler", - "allowImportingTsExtensions": true, - "verbatimModuleSyntax": true, - "noEmit": true, - - // Best practices - "strict": true, - "skipLibCheck": true, - "noFallthroughCasesInSwitch": true, - - // Some stricter flags (disabled by default) - "noUnusedLocals": false, - "noUnusedParameters": false, - "noPropertyAccessFromIndexSignature": false - } -} diff --git a/docs/configuration/index.md b/docs/configuration/index.md index c0e5b72..2be6ceb 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -37,6 +37,7 @@ Most settings can be reloaded without restarting PgDog. This allows you to tweak To make things simpler, all units of time are in milliseconds. For example, if you want to set the pool checkout timeout to 5 seconds, convert it to 5000ms instead: ```toml +[general] checkout_timeout = 5_000 ``` diff --git a/docs/configuration/pgdog.toml/mirroring.md b/docs/configuration/pgdog.toml/mirroring.md index 6dcd2b1..3731ea1 100644 --- a/docs/configuration/pgdog.toml/mirroring.md +++ b/docs/configuration/pgdog.toml/mirroring.md @@ -12,7 +12,7 @@ For example: [[mirroring]] source_db = "source" destination_db = "dest" -queue_depth = 500 +queue_length = 500 exposure = 0.1 ``` diff --git a/docs/configuration/pgdog.toml/plugins.md b/docs/configuration/pgdog.toml/plugins.md index 19524f9..62a162b 100644 --- a/docs/configuration/pgdog.toml/plugins.md +++ b/docs/configuration/pgdog.toml/plugins.md @@ -28,6 +28,7 @@ name is `router`, PgDog will look for `librouter.so` on Linux, `librouter.dll` o Additionally, you can pass the relative or absolute path to the shared library itself: ```toml +[[plugins]] name = "/opt/plugins/librouter.so" ``` diff --git a/docs/configuration/pgdog.toml/sharded_tables.md b/docs/configuration/pgdog.toml/sharded_tables.md index 61c9ea8..decf45f 100644 --- a/docs/configuration/pgdog.toml/sharded_tables.md +++ b/docs/configuration/pgdog.toml/sharded_tables.md @@ -15,7 +15,7 @@ The following configuration will match queries referring to this exact table and ```toml [[sharded_tables]] database = "prod" -table = "users" +name = "users" column = "id" data_type = "bigint" ``` diff --git a/docs/features/authentication.md b/docs/features/authentication.md index 0e7855e..5e00970 100644 --- a/docs/features/authentication.md +++ b/docs/features/authentication.md @@ -63,6 +63,10 @@ PgDog will expect clients connecting as `pgdog` to provide the password `hunter2 You can override the user and/or password PgDog uses to connect to Postgres by specifying `server_user` and `server_password` in the same configuration: ```toml +[[users]] +name = "pgdog" +password = "hunter2" +database = "pgdog" server_user = "bob" server_password = "opensesame" ``` diff --git a/docs/features/load-balancer/healthchecks.md b/docs/features/load-balancer/healthchecks.md index 6090b1f..84afbfb 100644 --- a/docs/features/load-balancer/healthchecks.md +++ b/docs/features/load-balancer/healthchecks.md @@ -81,6 +81,7 @@ Databases are automatically put back into the load balancer after a period of ti The amount of time the database is banned from serving traffic is controlled with the `ban_timeout` setting: ```toml +[general] ban_timeout = 300_000 # 5 minutes ``` @@ -101,7 +102,7 @@ By default, the load balancer gives the database a limited amount of time to ans This behavior is configurable with the `healthcheck_timeout` setting: ```toml -[global] +[general] healthcheck_timeout = 5_000 # 5 seconds in ms ``` diff --git a/docs/features/tls.md b/docs/features/tls.md index 098e2df..7872a2e 100644 --- a/docs/features/tls.md +++ b/docs/features/tls.md @@ -47,6 +47,7 @@ The default value for most PostgreSQL connection drivers is typically `prefer`, By default, PgDog will attempt to use TLS when connecting to PostgreSQL. This is configurable via a setting: ```toml +[general] tls_verify = "prefer" ``` @@ -62,6 +63,7 @@ This setting accepts almost identical values to the `sslmode` parameter used by If you use `verify_ca` or `verify_full` and your certificate is not signed by a well known CA, you can configure PgDog to validate it using your own certificate chain: ```toml +[general] tls_server_ca_certificate = "/path/to/ca/certificate.pem" ``` diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..aa71855 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,2 @@ +markdown-it-py +regex diff --git a/tests/test_code_blocks.py b/tests/test_code_blocks.py new file mode 100644 index 0000000..e153172 --- /dev/null +++ b/tests/test_code_blocks.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import glob +import re +import subprocess +from markdown_it import MarkdownIt +import sys + +from regex import sub +from regex.regex import Regex, RegexFlag +mdp = MarkdownIt() + +pattern = re.compile(r'(?msi)^(?P[`~]{3,})[ \t]*toml\b[^\n]*\r?\n(?P.*?)^(?P=fence)[ \t]*\r?$',) + +def verify(binary): + for file in glob.glob("docs/**/*.md", + recursive=True): + with open(file, "r") as f: + content = f.read() + print(f"Checking {file}") + tokens = mdp.parse(content) + for token in tokens: + if token.type == "fence" and token.info == "toml": + if "[[users]]" in token.content: + check_file(binary, "users", token.content) + elif "[lib]" in token.content: + pass + else: + check_file(binary, "pgdog", token.content) + +def check_file(binary, kind, content): + tmp = f"/tmp/pgdog_config_test.toml" + with open(tmp, "w") as f: + f.write(content) + if kind =="users": + arg = "--users" + else: + arg = "--config" + result = subprocess.run([binary, arg, "/tmp/pgdog_config_test.toml", "configcheck"]) + if result.returncode != 0: + print(f"{content}") + print(result.stderr) + exit(1) + +if __name__ == "__main__": + verify(sys.argv[1])