Skip to content

Commit d308abe

Browse files
ilyarioilyario
committed
Normalization pg password
1 parent 77373ce commit d308abe

3 files changed

Lines changed: 302 additions & 2 deletions

File tree

‎internal/connectors/postgresql.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ func (p *PostgreSQLSourceConnector) Connect(ctx context.Context) error {
8383
defer p.Unlock()
8484

8585
p.logger.Info("Connecting to PostgreSQL", "table", p.config.Table)
86-
conn, err := pgx.Connect(ctx, p.config.ConnectionString)
86+
conn, err := pgx.Connect(ctx, normalizePostgreSQLConnectionString(p.config.ConnectionString))
8787
if err != nil {
8888
p.RecordError("connect", "connection_error")
8989
p.logger.Error(err, "Failed to connect to PostgreSQL", "table", p.config.Table)
@@ -379,7 +379,7 @@ func (p *PostgreSQLSinkConnector) Connect(ctx context.Context) error {
379379
defer p.Unlock()
380380

381381
p.logger.Info("Connecting to PostgreSQL", "table", p.config.Table)
382-
conn, err := pgx.Connect(ctx, p.config.ConnectionString)
382+
conn, err := pgx.Connect(ctx, normalizePostgreSQLConnectionString(p.config.ConnectionString))
383383
if err != nil {
384384
p.RecordError("connect", "connection_error")
385385
p.logger.Error(err, "Failed to connect to PostgreSQL", "table", p.config.Table)

‎internal/connectors/postgresql_connstring.go‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
/*
2+
Copyright 2024.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package connectors
18+
19+
import (
20+
"strings"
21+
22+
"github.com/jackc/pgx/v5"
23+
)
24+
25+
// normalizePostgreSQLConnectionString returns a connection string that pgx can parse.
26+
// URL-form strings with special characters in the password (e.g. @, :, %) often fail
27+
// net/url parsing; those are converted to libpq key=value format where the raw password is preserved.
28+
func normalizePostgreSQLConnectionString(connStr string) string {
29+
connStr = strings.TrimSpace(connStr)
30+
if connStr == "" {
31+
return connStr
32+
}
33+
if _, err := pgx.ParseConfig(connStr); err == nil {
34+
return connStr
35+
}
36+
if libpq, ok := postgreSQLURLToLibpq(connStr); ok {
37+
if _, err := pgx.ParseConfig(libpq); err == nil {
38+
return libpq
39+
}
40+
}
41+
return connStr
42+
}
43+
44+
func postgreSQLURLToLibpq(connStr string) (string, bool) {
45+
scheme, rest, ok := strings.Cut(connStr, "://")
46+
if !ok {
47+
return "", false
48+
}
49+
switch strings.ToLower(scheme) {
50+
case "postgres", "postgresql":
51+
default:
52+
return "", false
53+
}
54+
55+
query := ""
56+
if idx := strings.Index(rest, "?"); idx >= 0 {
57+
query = rest[idx+1:]
58+
rest = rest[:idx]
59+
}
60+
61+
// Split userinfo from host first: password may contain '/' or '@'.
62+
at := strings.LastIndex(rest, "@")
63+
if at < 0 {
64+
return "", false
65+
}
66+
userinfo := rest[:at]
67+
hostPath := rest[at+1:]
68+
69+
dbname := ""
70+
hostport := hostPath
71+
if idx := strings.Index(hostPath, "/"); idx >= 0 {
72+
hostport = hostPath[:idx]
73+
dbname = hostPath[idx+1:]
74+
}
75+
76+
user, password, hasPassword := strings.Cut(userinfo, ":")
77+
78+
var parts []string
79+
host, port := hostport, ""
80+
if idx := strings.LastIndex(hostport, ":"); idx >= 0 {
81+
host = hostport[:idx]
82+
port = hostport[idx+1:]
83+
}
84+
if host != "" {
85+
parts = append(parts, "host="+host)
86+
if port != "" {
87+
parts = append(parts, "port="+port)
88+
}
89+
}
90+
if user != "" {
91+
parts = append(parts, "user="+user)
92+
}
93+
if hasPassword {
94+
parts = append(parts, "password="+quoteLibpqValue(password))
95+
}
96+
if dbname != "" {
97+
parts = append(parts, "dbname="+dbname)
98+
}
99+
for _, param := range strings.Split(query, "&") {
100+
param = strings.TrimSpace(param)
101+
if param != "" {
102+
parts = append(parts, param)
103+
}
104+
}
105+
if len(parts) == 0 {
106+
return "", false
107+
}
108+
return strings.Join(parts, " "), true
109+
}
110+
111+
func quoteLibpqValue(v string) string {
112+
if v == "" {
113+
return "''"
114+
}
115+
if !strings.ContainsAny(v, " \t'\\") {
116+
return v
117+
}
118+
var b strings.Builder
119+
b.WriteByte('\'')
120+
for _, r := range v {
121+
switch r {
122+
case '\'':
123+
b.WriteString(`\'`)
124+
case '\\':
125+
b.WriteString(`\\`)
126+
default:
127+
b.WriteRune(r)
128+
}
129+
}
130+
b.WriteByte('\'')
131+
return b.String()
132+
}

‎internal/connectors/postgresql_connstring_test.go‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
/*
2+
Copyright 2024.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package connectors
18+
19+
import (
20+
"testing"
21+
22+
"github.com/jackc/pgx/v5"
23+
"github.com/stretchr/testify/assert"
24+
"github.com/stretchr/testify/require"
25+
)
26+
27+
func TestPostgreSQLURLToLibpq(t *testing.T) {
28+
tests := []struct {
29+
name string
30+
input string
31+
want string
32+
wantOK bool
33+
}{
34+
{
35+
name: "password with at sign",
36+
input: "postgresql://com_devops:p@ss@db.example.net:6432/mydb?sslmode=require",
37+
want: "host=db.example.net port=6432 user=com_devops password=p@ss dbname=mydb sslmode=require",
38+
wantOK: true,
39+
},
40+
{
41+
name: "password with colon",
42+
input: "postgres://user:pa:ss@localhost:5432/db?sslmode=disable",
43+
want: "host=localhost port=5432 user=user password=pa:ss dbname=db sslmode=disable",
44+
wantOK: true,
45+
},
46+
{
47+
name: "password with percent",
48+
input: "postgresql://user:100%25off@localhost:5432/db",
49+
want: "host=localhost port=5432 user=user password=100%25off dbname=db",
50+
wantOK: true,
51+
},
52+
{
53+
name: "password with colon and slash",
54+
input: "postgresql://user:pa:ss/w0rd@host.example.com:5432/dbname?sslmode=require",
55+
want: "host=host.example.com port=5432 user=user password=pa:ss/w0rd dbname=dbname sslmode=require",
56+
wantOK: true,
57+
},
58+
{
59+
name: "password with spaces needs quoting",
60+
input: "postgresql://user:my pass@localhost:5432/db",
61+
want: "host=localhost port=5432 user=user password='my pass' dbname=db",
62+
wantOK: true,
63+
},
64+
{
65+
name: "libpq format unchanged",
66+
input: "host=localhost user=u password=secret dbname=db",
67+
wantOK: false,
68+
},
69+
{
70+
name: "no credentials",
71+
input: "postgresql://localhost/db",
72+
wantOK: false,
73+
},
74+
}
75+
76+
for _, tt := range tests {
77+
t.Run(tt.name, func(t *testing.T) {
78+
got, ok := postgreSQLURLToLibpq(tt.input)
79+
assert.Equal(t, tt.wantOK, ok)
80+
if tt.wantOK {
81+
assert.Equal(t, tt.want, got)
82+
}
83+
})
84+
}
85+
}
86+
87+
func TestNormalizePostgreSQLConnectionString(t *testing.T) {
88+
tests := []struct {
89+
name string
90+
input string
91+
}{
92+
{
93+
name: "password with at sign",
94+
input: "postgresql://com_devops:Secr@t!@c-c9q3r1b3d8imi0qgqpcc.rw.mdb.yandexcloud.net:6432/mpa_content_price_stock?sslmode=require",
95+
},
96+
{
97+
name: "password with colon and slash",
98+
input: "postgresql://user:pa:ss/w0rd@host.example.com:5432/dbname?sslmode=require",
99+
},
100+
{
101+
name: "already valid url",
102+
input: "postgresql://user:simple@localhost:5432/db?sslmode=disable",
103+
},
104+
{
105+
name: "libpq format",
106+
input: "host=localhost port=5432 user=user password=plain dbname=db sslmode=disable",
107+
},
108+
}
109+
110+
for _, tt := range tests {
111+
t.Run(tt.name, func(t *testing.T) {
112+
normalized := normalizePostgreSQLConnectionString(tt.input)
113+
_, err := pgx.ParseConfig(normalized)
114+
require.NoError(t, err, "normalized=%q", normalized)
115+
})
116+
}
117+
}
118+
119+
func TestNormalizePostgreSQLConnectionStringTrimsWhitespace(t *testing.T) {
120+
input := " postgresql://user:p@ss@localhost:5432/db?sslmode=disable \n"
121+
normalized := normalizePostgreSQLConnectionString(input)
122+
_, err := pgx.ParseConfig(normalized)
123+
require.NoError(t, err)
124+
}
125+
126+
func TestNormalizePostgreSQLConnectionString_ProductionPasswords(t *testing.T) {
127+
const (
128+
yandexHost = "c-c9q3r1b3d8imi0qgqpcc.rw.mdb.yandexcloud.net"
129+
user = "com_devops"
130+
)
131+
132+
tests := []struct {
133+
name string
134+
password string
135+
dbname string
136+
}{
137+
{
138+
name: "mpa content price stock",
139+
password: "yonE8fc[wJfNuoEs",
140+
dbname: "mpa_content_price_stock",
141+
},
142+
{
143+
name: "price core",
144+
password: "n)PRLB}aHeLd[r1G",
145+
dbname: "price_core",
146+
},
147+
}
148+
149+
for _, tt := range tests {
150+
t.Run(tt.name, func(t *testing.T) {
151+
input := "postgresql://" + user + ":" + tt.password + "@" + yandexHost + ":6432/" + tt.dbname + "?sslmode=require"
152+
153+
_, err := pgx.ParseConfig(input)
154+
require.Error(t, err, "raw URL with special chars in password should not parse")
155+
156+
normalized := normalizePostgreSQLConnectionString(input)
157+
require.NotEqual(t, input, normalized, "expected conversion to libpq format")
158+
159+
cfg, err := pgx.ParseConfig(normalized)
160+
require.NoError(t, err, "normalized=%q", normalized)
161+
assert.Equal(t, user, cfg.User)
162+
assert.Equal(t, tt.password, cfg.Password)
163+
assert.Equal(t, yandexHost, cfg.Host)
164+
assert.Equal(t, uint16(6432), cfg.Port)
165+
assert.Equal(t, tt.dbname, cfg.Database)
166+
})
167+
}
168+
}

0 commit comments

Comments
 (0)