From ba2ae466be9894580112f6a26e5e206219b4accb Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 22 Sep 2025 21:19:59 -0700 Subject: [PATCH 1/4] Add doc tests --- .github/workflows/ci.yaml | 65 ++++----- ci/bash/verify_snippets.bash | 45 ------ ci/bun/bun.lock | 29 ---- ci/bun/package.json | 12 -- ci/bun/scripts/collectSnippets.ts | 131 ------------------ ci/bun/tsconfig.json | 27 ---- docs/configuration/index.md | 1 + docs/configuration/pgdog.toml/mirroring.md | 2 +- docs/configuration/pgdog.toml/plugins.md | 1 + .../pgdog.toml/sharded_tables.md | 2 +- docs/features/authentication.md | 4 + docs/features/load-balancer/healthchecks.md | 3 +- docs/features/tls.md | 2 + tests/requirements.txt | 1 + tests/test_code_blocks.py | 46 ++++++ 15 files changed, 92 insertions(+), 279 deletions(-) delete mode 100644 ci/bash/verify_snippets.bash delete mode 100644 ci/bun/bun.lock delete mode 100644 ci/bun/package.json delete mode 100644 ci/bun/scripts/collectSnippets.ts delete mode 100644 ci/bun/tsconfig.json create mode 100644 tests/requirements.txt create mode 100644 tests/test_code_blocks.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eb9323c..52f4ad3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,35 +1,36 @@ name: CI on: - push: - branches: [main] - pull_request: - branches: [main] + push: + branches: [main] + pull_request: + branches: [main] jobs: - verify-tomls: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Bun - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - - name: Install dependencies - run: bun install - working-directory: ci/bun - - - name: Collect snippets - run: bun run ci/bun/scripts/collectSnippets.ts - - - name: Verify snippets - run: bash ci/bash/verify_snippets.bash - - - name: Upload log artifact - if: always() - uses: actions/upload-artifact@v4 - with: - name: verification-log - path: verification-log.json + verify-tomls: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Checkout another public repo + uses: actions/checkout@v4 + with: + repository: pgdogdev/pgdog + ref: main + path: pgdog-source + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - name: Install CMake 3.31 + run: | + sudo apt update && sudo apt install mold -y + sudo apt remove cmake + sudo pip3 install cmake==3.31.6 + cmake --version + - name: Build + run: cargo build + working-directory: pgdog-source + - name: Install test dependencies + run: sudo pip3 install -r tests/requirements.txt + - name: Run tests + run: python3 tests/test_code_blocks.py pgdog-source/target/debug/pgdog diff --git a/ci/bash/verify_snippets.bash b/ci/bash/verify_snippets.bash deleted file mode 100644 index 54ca4ef..0000000 --- a/ci/bash/verify_snippets.bash +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -set -euo pipefail - -OUTPUT_DIRECTORY="${PWD}/ci/tmp" - -# DOCKER_IMAGE = "ghcr.io/pgdogdev/pgdog:main"; <- change when cache if fixed -DOCKER_IMAGE="ghcr.io/pgdogdev/pgdog:main@sha256:3036d2ac7b684643dd187c42971f003f9d76e5f54cd129dcba742c309d7debd0" - -# nuke previous snippets -rm -rf "$OUTPUT_DIRECTORY" -mkdir -p "$OUTPUT_DIRECTORY" - -# Pull image once -docker pull "$DOCKER_IMAGE" - -# Run one container and do all checks inside it -docker run --rm \ - -v "$OUTPUT_DIRECTORY":/ci/tmp \ - "$DOCKER_IMAGE" \ - sh -euc ' - ANY_FAILED=0 - - for toml in /ci/tmp/*.toml; do - [ -f "$toml" ] || continue - - orig=$(head -n1 "$toml" | sed "s/# file: //") - line=$(head -n2 "$toml" | tail -n1 | sed "s/# line_number: //") - - if pgdog configcheck --config "$toml" >/dev/null 2>&1 \ - || pgdog configcheck --users "$toml" >/dev/null 2>&1; then - echo "${orig}:${line} verified successfully" - else - echo "Validation failed for ${orig}:${line}" >&2 - ANY_FAILED=1 - fi - done - - if [ "$ANY_FAILED" -eq 1 ]; then - echo "Some snippets failed verification!" >&2 - exit 1 - else - echo "All snippets verified successfully!" - exit 0 - fi - ' diff --git a/ci/bun/bun.lock b/ci/bun/bun.lock deleted file mode 100644 index 7b784bd..0000000 --- a/ci/bun/bun.lock +++ /dev/null @@ -1,29 +0,0 @@ -{ - "lockfileVersion": 1, - "workspaces": { - "": { - "name": "bun", - "devDependencies": { - "@types/bun": "latest" - }, - "peerDependencies": { - "typescript": "^5" - } - } - }, - "packages": { - "@types/bun": ["@types/bun@1.2.19", "", { "dependencies": { "bun-types": "1.2.19" } }, "sha512-d9ZCmrH3CJ2uYKXQIUuZ/pUnTqIvLDS0SK7pFmbx8ma+ziH/FRMoAq5bYpRG7y+w1gl+HgyNZbtqgMq4W4e2Lg=="], - - "@types/node": ["@types/node@24.1.0", "", { "dependencies": { "undici-types": "~7.8.0" } }, "sha512-ut5FthK5moxFKH2T1CUOC6ctR67rQRvvHdFLCD2Ql6KXmMuCrjsSsRI9UsLCm9M18BMwClv4pn327UvB7eeO1w=="], - - "@types/react": ["@types/react@19.1.9", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-WmdoynAX8Stew/36uTSVMcLJJ1KRh6L3IZRx1PZ7qJtBqT3dYTgyDTx8H1qoRghErydW7xw9mSJ3wS//tCRpFA=="], - - "bun-types": ["bun-types@1.2.19", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-uAOTaZSPuYsWIXRpj7o56Let0g/wjihKCkeRqUBhlLVM/Bt+Fj9xTo+LhC1OV1XDaGkz4hNC80et5xgy+9KTHQ=="], - - "csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="], - - "typescript": ["typescript@5.9.2", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A=="], - - "undici-types": ["undici-types@7.8.0", "", {}, "sha512-9UJ2xGDvQ43tYyVMpuHlsgApydB8ZKfVYTsLDhXkFL/6gfkp+U8xTGdh8pMJv1SpZna0zxG1DwsKZsreLbXBxw=="] - } -} diff --git a/ci/bun/package.json b/ci/bun/package.json deleted file mode 100644 index 5808d97..0000000 --- a/ci/bun/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "bun", - "module": "index.ts", - "type": "module", - "private": true, - "devDependencies": { - "@types/bun": "latest" - }, - "peerDependencies": { - "typescript": "^5" - } -} \ No newline at end of file diff --git a/ci/bun/scripts/collectSnippets.ts b/ci/bun/scripts/collectSnippets.ts deleted file mode 100644 index 4c83e17..0000000 --- a/ci/bun/scripts/collectSnippets.ts +++ /dev/null @@ -1,131 +0,0 @@ -// ----------------------------------------------------------------------------- -// ----- Constants ------------------------------------------------------------- - -const TOML_SNIPPET_REGEX = /^([ \t]*)```toml[^\n]*\r?\n([\s\S]*?)^\1```/gm; - -const DOCS_ROOT_DIRECTORY = `${process.cwd()}/docs`; -const OUTPUT_DIRECTORY = `${process.cwd()}/ci/tmp`; - -const SnippetsByMd5 = new Map(); - -// ----------------------------------------------------------------------------- -// ----- Invoke main() --------------------------------------------------------- - -main(); - -// ----------------------------------------------------------------------------- -// ----- Main ------------------------------------------------------------------ - -async function main() { - await setup(); - - // Scan docs for TOML snippets - const snippets = await extractTomlSnippets(); - - // Create one temporary {md5}.toml file for each snippet - await Promise.all(snippets.map(createFileForSnippet)); -} - -// ----------------------------------------------------------------------------- -// ----- Types ----------------------------------------------------------------- - -type Snippet = { - file: string; - content: string; - line: number; -}; - -// ----------------------------------------------------------------------------- -// ----- Helpers --------------------------------------------------------------- - -async function extractTomlSnippets() { - const glob = new Bun.Glob("**/*.md"); - const paths = glob.scan(DOCS_ROOT_DIRECTORY); - - const snippets: Snippet[] = []; - - for await (const relativePath of paths) { - const fullPath = `${DOCS_ROOT_DIRECTORY}/${relativePath}`; - const fileText = await Bun.file(fullPath).text(); - - while (true) { - const match = TOML_SNIPPET_REGEX.exec(fileText); - if (match === null) { - break; - } - - const codeIndent = match[1]; - const deindentedContent = match[2] - .split(/\r?\n/) - .map((line) => - line.startsWith(codeIndent) ? line.slice(codeIndent.length) : line, - ) - .join("\n") - .trim(); - - const startIndex = match.index; - const startLineNumber = fileText - .slice(0, startIndex) - .split(/\r?\n/).length; - - const snippet = { - file: relativePath, - content: deindentedContent, - line: startLineNumber, - }; - - const md5 = hash(snippet); - - SnippetsByMd5.set(md5, snippet); - snippets.push(snippet); - } - } - - return snippets; -} - -// ----------------------------------------------------------------------------- - -function hash(snippet: Snippet): string { - const hasher = new Bun.CryptoHasher("md5"); - hasher.update(JSON.stringify(snippet)); - const md5 = hasher.digest("hex"); - return md5; -} - -// ----------------------------------------------------------------------------- - -async function setup() { - await Bun.$`mkdir -p ${OUTPUT_DIRECTORY}`; - await Bun.$`rm -rf ${OUTPUT_DIRECTORY}`.nothrow(); -} - -// ----------------------------------------------------------------------------- - -async function createFileForSnippet(snippet: Snippet) { - const prefix = guessType(snippet.content); - const md5 = hash(snippet); - const filename = `${prefix}_${md5}.toml`; - const outputFilePath = `${OUTPUT_DIRECTORY}/${filename}`; - - const content = [ - `# file: ${snippet.file}`, - `# line_number: ${snippet.line}`, - "", - `${snippet.content}`, - "", - ].join("\n"); - - await Bun.write(outputFilePath, content); -} - -// ----------------------------------------------------------------------------- - -type ConfigOrUsers = "config" | "users"; - -function guessType(snippetContent: string): ConfigOrUsers { - return snippetContent.includes("[[users]]") ? "users" : "config"; -} - -// ----------------------------------------------------------------------------- -// ----------------------------------------------------------------------------- diff --git a/ci/bun/tsconfig.json b/ci/bun/tsconfig.json deleted file mode 100644 index 238655f..0000000 --- a/ci/bun/tsconfig.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "compilerOptions": { - // Enable latest features - "lib": ["ESNext", "DOM"], - "target": "ESNext", - "module": "ESNext", - "moduleDetection": "force", - "jsx": "react-jsx", - "allowJs": true, - - // Bundler mode - "moduleResolution": "bundler", - "allowImportingTsExtensions": true, - "verbatimModuleSyntax": true, - "noEmit": true, - - // Best practices - "strict": true, - "skipLibCheck": true, - "noFallthroughCasesInSwitch": true, - - // Some stricter flags (disabled by default) - "noUnusedLocals": false, - "noUnusedParameters": false, - "noPropertyAccessFromIndexSignature": false - } -} diff --git a/docs/configuration/index.md b/docs/configuration/index.md index c0e5b72..2be6ceb 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -37,6 +37,7 @@ Most settings can be reloaded without restarting PgDog. This allows you to tweak To make things simpler, all units of time are in milliseconds. For example, if you want to set the pool checkout timeout to 5 seconds, convert it to 5000ms instead: ```toml +[general] checkout_timeout = 5_000 ``` diff --git a/docs/configuration/pgdog.toml/mirroring.md b/docs/configuration/pgdog.toml/mirroring.md index 6dcd2b1..3731ea1 100644 --- a/docs/configuration/pgdog.toml/mirroring.md +++ b/docs/configuration/pgdog.toml/mirroring.md @@ -12,7 +12,7 @@ For example: [[mirroring]] source_db = "source" destination_db = "dest" -queue_depth = 500 +queue_length = 500 exposure = 0.1 ``` diff --git a/docs/configuration/pgdog.toml/plugins.md b/docs/configuration/pgdog.toml/plugins.md index 19524f9..62a162b 100644 --- a/docs/configuration/pgdog.toml/plugins.md +++ b/docs/configuration/pgdog.toml/plugins.md @@ -28,6 +28,7 @@ name is `router`, PgDog will look for `librouter.so` on Linux, `librouter.dll` o Additionally, you can pass the relative or absolute path to the shared library itself: ```toml +[[plugins]] name = "/opt/plugins/librouter.so" ``` diff --git a/docs/configuration/pgdog.toml/sharded_tables.md b/docs/configuration/pgdog.toml/sharded_tables.md index 61c9ea8..decf45f 100644 --- a/docs/configuration/pgdog.toml/sharded_tables.md +++ b/docs/configuration/pgdog.toml/sharded_tables.md @@ -15,7 +15,7 @@ The following configuration will match queries referring to this exact table and ```toml [[sharded_tables]] database = "prod" -table = "users" +name = "users" column = "id" data_type = "bigint" ``` diff --git a/docs/features/authentication.md b/docs/features/authentication.md index 0e7855e..5e00970 100644 --- a/docs/features/authentication.md +++ b/docs/features/authentication.md @@ -63,6 +63,10 @@ PgDog will expect clients connecting as `pgdog` to provide the password `hunter2 You can override the user and/or password PgDog uses to connect to Postgres by specifying `server_user` and `server_password` in the same configuration: ```toml +[[users]] +name = "pgdog" +password = "hunter2" +database = "pgdog" server_user = "bob" server_password = "opensesame" ``` diff --git a/docs/features/load-balancer/healthchecks.md b/docs/features/load-balancer/healthchecks.md index 6090b1f..84afbfb 100644 --- a/docs/features/load-balancer/healthchecks.md +++ b/docs/features/load-balancer/healthchecks.md @@ -81,6 +81,7 @@ Databases are automatically put back into the load balancer after a period of ti The amount of time the database is banned from serving traffic is controlled with the `ban_timeout` setting: ```toml +[general] ban_timeout = 300_000 # 5 minutes ``` @@ -101,7 +102,7 @@ By default, the load balancer gives the database a limited amount of time to ans This behavior is configurable with the `healthcheck_timeout` setting: ```toml -[global] +[general] healthcheck_timeout = 5_000 # 5 seconds in ms ``` diff --git a/docs/features/tls.md b/docs/features/tls.md index 098e2df..7872a2e 100644 --- a/docs/features/tls.md +++ b/docs/features/tls.md @@ -47,6 +47,7 @@ The default value for most PostgreSQL connection drivers is typically `prefer`, By default, PgDog will attempt to use TLS when connecting to PostgreSQL. This is configurable via a setting: ```toml +[general] tls_verify = "prefer" ``` @@ -62,6 +63,7 @@ This setting accepts almost identical values to the `sslmode` parameter used by If you use `verify_ca` or `verify_full` and your certificate is not signed by a well known CA, you can configure PgDog to validate it using your own certificate chain: ```toml +[general] tls_server_ca_certificate = "/path/to/ca/certificate.pem" ``` diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..d0a92d0 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1 @@ +markdown-it-py==4.0.0 diff --git a/tests/test_code_blocks.py b/tests/test_code_blocks.py new file mode 100644 index 0000000..10d7d06 --- /dev/null +++ b/tests/test_code_blocks.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import glob +import re +import subprocess +from markdown_it import MarkdownIt +import sys + +from regex import sub +from regex.regex import Regex, RegexFlag +mdp = MarkdownIt() + +pattern = re.compile(r'(?msi)^(?P[`~]{3,})[ \t]*toml\b[^\n]*\r?\n(?P.*?)^(?P=fence)[ \t]*\r?$',) + +def verify(binary): + for file in glob.glob("**/*.md", + recursive=True): + with open(file, "r") as f: + content = f.read() + print(f"Checking {file}") + tokens = mdp.parse(content) + for token in tokens: + if token.type == "fence" and token.info == "toml": + if "[[users]]" in token.content: + check_file(binary, "users", token.content) + elif "[lib]" in token.content: + pass + else: + check_file(binary, "pgdog", token.content) + +def check_file(binary, kind, content): + tmp = f"/tmp/pgdog_config_test.toml" + with open(tmp, "w") as f: + f.write(content) + if kind =="users": + arg = "--users" + else: + arg = "--config" + result = subprocess.run([binary, arg, "/tmp/pgdog_config_test.toml", "configcheck"]) + if result.returncode != 0: + print(f"{content}") + print(result.stderr) + exit(1) + +if __name__ == "__main__": + verify(sys.argv[1]) From a29bc93fbb6a63ad15eab63981226288a80ba5e0 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 22 Sep 2025 21:24:49 -0700 Subject: [PATCH 2/4] Faster runner --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 52f4ad3..1899b73 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,7 +6,7 @@ on: branches: [main] jobs: verify-tomls: - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 30 steps: - name: Checkout code From 66ae2bf606a3f519976c6553f57cb9048ba300e2 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 22 Sep 2025 21:25:39 -0700 Subject: [PATCH 3/4] fix test --- tests/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index d0a92d0..aa71855 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1 +1,2 @@ -markdown-it-py==4.0.0 +markdown-it-py +regex From ef80ab96697f35ce8bac9115433491780180ab2e Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Mon, 22 Sep 2025 21:32:04 -0700 Subject: [PATCH 4/4] Dir --- tests/test_code_blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_code_blocks.py b/tests/test_code_blocks.py index 10d7d06..e153172 100644 --- a/tests/test_code_blocks.py +++ b/tests/test_code_blocks.py @@ -13,7 +13,7 @@ pattern = re.compile(r'(?msi)^(?P[`~]{3,})[ \t]*toml\b[^\n]*\r?\n(?P.*?)^(?P=fence)[ \t]*\r?$',) def verify(binary): - for file in glob.glob("**/*.md", + for file in glob.glob("docs/**/*.md", recursive=True): with open(file, "r") as f: content = f.read()