From d8b727cff017815def4e25e277b1f1c5ea2ed4a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C5=82eczek?= Date: Thu, 5 Mar 2026 12:52:54 +0100 Subject: [PATCH] test(refactor): add withToxiproxy* in withTools This commit introduces toxiproxy in withTools adding the following helpers: * withToxiproxyServer * withToxiproxyProxy * withToxiproxyPgProxy --- nix/tools/withTools.nix | 123 +++++++++++++++++++++++++++++++++++++--- test/io/conftest.py | 6 +- test/io/test_auth.py | 2 +- test/io/test_io.py | 2 +- 4 files changed, 122 insertions(+), 11 deletions(-) diff --git a/nix/tools/withTools.nix b/nix/tools/withTools.nix index 094f13b359..ea629715bd 100644 --- a/nix/tools/withTools.nix +++ b/nix/tools/withTools.nix @@ -8,6 +8,7 @@ , python3Packages , writeText , writers +, toxiproxy }: let withTmpDb = @@ -54,6 +55,8 @@ let export PGDATA="$tmpdir/db" export PGHOST="$tmpdir/socket" + PGPORT=$(${randomPort}) + export PGPORT export PGUSER export PGDATABASE export PGRST_DB_SCHEMAS @@ -61,9 +64,16 @@ let export PGOPTIONS HBA_FILE="$tmpdir/pg_hba.conf" - echo "local $PGDATABASE some_protected_user password" > "$HBA_FILE" - echo "local $PGDATABASE all trust" >> "$HBA_FILE" - echo "local replication all trust" >> "$HBA_FILE" + { + echo "local $PGDATABASE some_protected_user password" + echo "local $PGDATABASE all trust" + echo "local replication all trust" + echo "host $PGDATABASE some_protected_user localhost password" + echo "host $PGDATABASE all localhost trust" + } >> "$HBA_FILE" + + UNIX_PGHOST="$PGHOST" + export TCP_PGHOST="localhost" log "Initializing database cluster..." # We try to make the database cluster as independent as possible from the host @@ -80,7 +90,7 @@ let # On MacOS, it's 104 chars # See: https://serverfault.com/questions/641347/check-if-a-path-exceeds-maximum-for-unix-domain-socket - pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $PGHOST -c log_statement=\"all\" " \ + pg_ctl -l "$tmpdir/db.log" -w start -o "-F -c listen_addresses=\"$TCP_PGHOST\" -c hba_file=$HBA_FILE -k $UNIX_PGHOST -c log_statement=\"all\" " \ >> "$setuplog" log "Creating a minimally privileged $PGUSER connection role..." @@ -93,6 +103,7 @@ let replica_slot="replica_$RANDOM" replica_dir="$tmpdir/$replica_slot" replica_host="$tmpdir/socket_$replica_slot" + replica_port=$(${randomPort}) mkdir -p "$replica_host" @@ -106,15 +117,16 @@ let log "Starting replica on $replica_host" # We set a low max_standby_streaming_delay to make the replication conflict fail faster in tests (otherwise it waits for the default 30s) - pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c listen_addresses=\"\" -c hba_file=$HBA_FILE -k $replica_host -c log_statement=\"all\" -c max_standby_streaming_delay=\"3s\" " \ + pg_ctl -D "$replica_dir" -l "$replica_dblog" -w start -o "-F -c listen_addresses=\"$TCP_PGHOST\" -c port=$replica_port -c hba_file=$HBA_FILE -k $replica_host -c log_statement=\"all\" -c max_standby_streaming_delay=\"3s\" " \ >> "$setuplog" >&2 echo "${commandName}: Replica enabled. You can connect to it with: psql 'postgres:///$PGDATABASE?host=$replica_host' -U postgres" >&2 echo "${commandName}: You can tail the replica logs with: tail -f $replica_dblog" export PGREPLICAHOST="$replica_host" + export PGREPLICAPORT="$replica_port" export PGREPLICASLOT="$replica_slot" - export PGRST_DB_URI="postgres:///$PGDATABASE?host=$PGREPLICAHOST,$PGHOST" + export PGRST_DB_URI="postgres:///$PGDATABASE?host=$PGREPLICAHOST,$PGHOST&port=$replica_port,$PGPORT" fi # shellcheck disable=SC2329 @@ -372,6 +384,100 @@ let libraries = [ python3Packages.pandas python3Packages.tabulate python3Packages.psutil ]; } (builtins.readFile ./monitor_pid.py); + + randomPort = + writers.writePython3 "postgrest-random-port" + { } + '' + import socket + s = socket.socket() + s.bind(("127.0.0.1", 0)) + print(s.getsockname()[1]) + s.close() + ''; + + withToxiproxyProxy = + checkedShellScript + { + name = "postgrest-with-toxiproxy-proxy"; + docs = "Run with Toxiproxy proxy created."; + args = + [ + "ARG_POSITIONAL_SINGLE([command], [Command to run])" + "ARG_LEFTOVERS([command arguments])" + "ARG_OPTIONAL_SINGLE([listen], [l], [Proxy will listen on this address])" + "ARG_OPTIONAL_SINGLE([upstream], [u], [Proxy will forward to this address])" + ]; + positionalCompletion = "_command"; + workingDir = "/"; + withPath = [ toxiproxy ]; + } + '' + proxyname="tp$RANDOM" + toxiproxy-cli create -l "$_arg_listen" -u "$_arg_upstream" "$proxyname" + + # shellcheck disable=SC2329 + stop () { + toxiproxy-cli delete "$proxyname" || true + } + trap stop EXIT + + (TOXI_PROXY_NAME="$proxyname" "$_arg_command" "''${_arg_leftovers[@]}") + ''; + + withToxiproxyPg = + checkedShellScript + { + name = "postgrest-with-toxiproxy-pg"; + docs = "Run with a Toxiproxy proxy to PostgreSQL."; + args = + [ + "ARG_POSITIONAL_SINGLE([command], [Command to run])" + "ARG_LEFTOVERS([command arguments])" + "ARG_USE_ENV([TCP_PGHOST], [], [PG host name])" + "ARG_USE_ENV([PGPORT], [], [PG port])" + ]; + positionalCompletion = "_command"; + workingDir = "/"; + } + '' + proxy_port=''$(${randomPort}) + + ${withToxiproxy} ${withToxiproxyProxy} -l "$TCP_PGHOST:$proxy_port" -u "$TCP_PGHOST:$PGPORT" \ + env "TOXI_PGPORT=$proxy_port" "$_arg_command" "''${_arg_leftovers[@]}" + ''; + + withToxiproxy = + checkedShellScript + { + name = "postgrest-with-toxiproxy"; + docs = "Run with toxiproxy-server"; + args = + [ + "ARG_POSITIONAL_SINGLE([command], [Command to run])" + "ARG_LEFTOVERS([command arguments])" + ]; + positionalCompletion = "_command"; + workingDir = "/"; + withPath = [ toxiproxy ]; + } + '' + if ! test -v TOXI_PROXY; then + export TOXI_PROXY="" + LOG_LEVEL=error toxiproxy-server& + TOXIPROXY_PID=$! + sleep 1 # give the server a moment to start + + # shellcheck disable=SC2329 + stop () { + kill "$TOXIPROXY_PID" || true + wait "$TOXIPROXY_PID" || true + } + trap stop EXIT + fi + ("$_arg_command" "''${_arg_leftovers[@]}") + ''; + in buildToolbox { @@ -380,11 +486,12 @@ buildToolbox inherit withGit withPgAll - withPgrst; + withPgrst + withToxiproxyPg; } // builtins.listToAttrs ( # Create a `postgrest-with-pg-` for each PostgreSQL version builtins.map (pg: { inherit (pg) name; value = withTmpDb pg; }) postgresqlVersions ); # make latest withPg available for other nix files - extra = { inherit withPg; }; + extra = { inherit withPg withToxiproxyPg; }; } diff --git a/test/io/conftest.py b/test/io/conftest.py index b6a068222f..dc42a9e0da 100644 --- a/test/io/conftest.py +++ b/test/io/conftest.py @@ -9,8 +9,9 @@ def dburi(): "Postgres database connection URI." dbname = os.environ["PGDATABASE"] host = os.environ["PGHOST"] + port = os.environ["PGPORT"] user = os.environ["PGUSER"] - return f"postgresql://?dbname={dbname}&host={host}&user={user}".encode() + return f"postgresql://?dbname={dbname}&host={host}&port={port}&user={user}".encode() @pytest.fixture @@ -19,6 +20,7 @@ def baseenv(): return { "PGDATABASE": os.environ["PGDATABASE"], "PGHOST": os.environ["PGHOST"], + "PGPORT": os.environ["PGPORT"], "PGUSER": os.environ["PGUSER"], } @@ -51,6 +53,7 @@ def replicaenv(defaultenv): **defaultenv, **conf, "PGHOST": os.environ["PGREPLICAHOST"] + "," + os.environ["PGHOST"], + "PGPORT": os.environ["PGREPLICAPORT"] + "," + os.environ["PGPORT"], "PGREPLICASLOT": os.environ["PGREPLICASLOT"], }, } @@ -76,6 +79,7 @@ def metapostgrest(): env = { "PGDATABASE": os.environ["PGDATABASE"], "PGHOST": os.environ["PGHOST"], + "PGPORT": os.environ["PGPORT"], "PGUSER": role, "PGRST_DB_ANON_ROLE": role, "PGRST_DB_CONFIG": "true", diff --git a/test/io/test_auth.py b/test/io/test_auth.py index 002fe46992..892b41b074 100644 --- a/test/io/test_auth.py +++ b/test/io/test_auth.py @@ -162,7 +162,7 @@ def test_jwt_errors(defaultenv): def test_fail_with_invalid_password(defaultenv): "Connecting with an invalid password should fail without retries." - uri = f'postgresql://?dbname={defaultenv["PGDATABASE"]}&host={defaultenv["PGHOST"]}&user=some_protected_user&password=invalid_pass' + uri = f'postgresql://?dbname={defaultenv["PGDATABASE"]}&host={defaultenv["PGHOST"]}&port={defaultenv["PGPORT"]}&user=some_protected_user&password=invalid_pass' env = {**defaultenv, "PGRST_DB_URI": uri} with run(env=env, wait_for_readiness=False) as postgrest: exitCode = wait_until_exit(postgrest) diff --git a/test/io/test_io.py b/test/io/test_io.py index 4b8c9f9acf..53f1b8dbc4 100644 --- a/test/io/test_io.py +++ b/test/io/test_io.py @@ -2022,7 +2022,7 @@ def test_log_listener_connection_start(defaultenv): # Check for the listener start message containing host and port # Do not check if pg version is displayed properly as it is tricky to test it assert any( - f'"{defaultenv["PGHOST"]}:5432" and listening for database notifications on the "pgrst" channel' + f'"{defaultenv["PGHOST"]}:{defaultenv["PGPORT"]}" and listening for database notifications on the "pgrst" channel' in line for line in output )