From 9233be2291cc4f4bca4c2ee0738061d05cb26edd Mon Sep 17 00:00:00 2001 From: Jonathan Haas Date: Wed, 15 Apr 2026 15:21:31 -0700 Subject: [PATCH 1/2] Renew and track Vault DB leases --- internal/connectors/vaultdb/client.go | 71 ++++++++++++ internal/connectors/vaultdb/client_test.go | 18 ++++ internal/connectors/vaultdb/connector.go | 59 +++++++++- internal/connectors/vaultdb/connector_test.go | 102 ++++++++++++++++++ 4 files changed, 246 insertions(+), 4 deletions(-) diff --git a/internal/connectors/vaultdb/client.go b/internal/connectors/vaultdb/client.go index f23c7db..0465bd8 100644 --- a/internal/connectors/vaultdb/client.go +++ b/internal/connectors/vaultdb/client.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "math" "net/http" "strings" "time" @@ -19,6 +20,7 @@ type HTTPClientConfig struct { Token string Namespace string Client *http.Client + RenewRetry resilience.RetryConfig RevokeRetry resilience.RetryConfig } @@ -27,6 +29,7 @@ type HTTPClient struct { token string namespace string client *http.Client + renewRetry resilience.RetryConfig revokeRetry resilience.RetryConfig } @@ -45,11 +48,22 @@ func NewHTTPClient(cfg HTTPClientConfig) *HTTPClient { if revokeRetry.MaxDelay == 0 { revokeRetry.MaxDelay = time.Second } + renewRetry := cfg.RenewRetry + if renewRetry.MaxAttempts == 0 { + renewRetry.MaxAttempts = 3 + } + if renewRetry.InitialDelay == 0 { + renewRetry.InitialDelay = 100 * time.Millisecond + } + if renewRetry.MaxDelay == 0 { + renewRetry.MaxDelay = time.Second + } return &HTTPClient{ baseURL: strings.TrimRight(cfg.BaseURL, "/"), token: cfg.Token, namespace: cfg.Namespace, client: client, + renewRetry: renewRetry, revokeRetry: revokeRetry, } } @@ -79,6 +93,7 @@ func (c *HTTPClient) GenerateCredentials(ctx context.Context, role string) (*Lea var payload struct { LeaseID string `json:"lease_id"` LeaseDuration int64 `json:"lease_duration"` + Renewable bool `json:"renewable"` Data struct { Username string `json:"username"` Password string `json:"password"` @@ -92,15 +107,71 @@ func (c *HTTPClient) GenerateCredentials(ctx context.Context, role string) (*Lea Password: payload.Data.Password, LeaseID: payload.LeaseID, LeaseDuration: time.Duration(payload.LeaseDuration) * time.Second, + Renewable: payload.Renewable, }, nil } +func (c *HTTPClient) RenewLease(ctx context.Context, leaseID string, increment time.Duration) (*LeaseCredentials, error) { + return resilience.RetryValue(ctx, c.renewRetry, func(ctx context.Context) (*LeaseCredentials, error) { + return c.renewLeaseOnce(ctx, leaseID, increment) + }) +} + func (c *HTTPClient) RevokeLease(ctx context.Context, leaseID string) error { return resilience.Retry(ctx, c.revokeRetry, func(ctx context.Context) error { return c.revokeLeaseOnce(ctx, leaseID) }) } +func (c *HTTPClient) renewLeaseOnce(ctx context.Context, leaseID string, increment time.Duration) (*LeaseCredentials, error) { + payload, err := json.Marshal(map[string]any{ + "lease_id": leaseID, + "increment": int(math.Ceil(increment.Seconds())), + }) + if err != nil { + return nil, resilience.Permanent(err) + } + request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/sys/leases/renew", bytes.NewReader(payload)) + if err != nil { + return nil, resilience.Permanent(err) + } + c.applyHeaders(request) + request.Header.Set("Content-Type", "application/json") + + response, err := c.client.Do(request) + if err != nil { + return nil, err + } + defer func() { + _ = response.Body.Close() + }() + body, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + if response.StatusCode >= 400 { + wrapped := fmt.Errorf("%w: vault renew lease returned %d: %s", core.ErrForbidden, response.StatusCode, string(body)) + if response.StatusCode == http.StatusTooManyRequests || response.StatusCode >= http.StatusInternalServerError { + return nil, wrapped + } + return nil, resilience.Permanent(wrapped) + } + + var renewed struct { + LeaseID string `json:"lease_id"` + LeaseDuration int64 `json:"lease_duration"` + Renewable bool `json:"renewable"` + } + if err := json.Unmarshal(body, &renewed); err != nil { + return nil, resilience.Permanent(err) + } + return &LeaseCredentials{ + LeaseID: renewed.LeaseID, + LeaseDuration: time.Duration(renewed.LeaseDuration) * time.Second, + Renewable: renewed.Renewable, + }, nil +} + func (c *HTTPClient) revokeLeaseOnce(ctx context.Context, leaseID string) error { body, err := json.Marshal(map[string]string{"lease_id": leaseID}) if err != nil { diff --git a/internal/connectors/vaultdb/client_test.go b/internal/connectors/vaultdb/client_test.go index c56dfdd..c598829 100644 --- a/internal/connectors/vaultdb/client_test.go +++ b/internal/connectors/vaultdb/client_test.go @@ -25,8 +25,19 @@ func TestHTTPClient_GenerateCredentialsAndRevokeLease(t *testing.T) { _, _ = w.Write([]byte(`{ "lease_id":"database/creds/analytics_ro/123", "lease_duration":600, + "renewable":true, "data":{"username":"dyn-user","password":"dyn-pass"} }`)) + case "/v1/sys/leases/renew": + body, _ := io.ReadAll(r.Body) + if string(body) != `{"increment":1800,"lease_id":"database/creds/analytics_ro/123"}` { + t.Fatalf("renew body = %s, want lease renew payload", string(body)) + } + _, _ = w.Write([]byte(`{ + "lease_id":"database/creds/analytics_ro/123", + "lease_duration":1800, + "renewable":true + }`)) case "/v1/sys/leases/revoke": body, _ := io.ReadAll(r.Body) if string(body) != `{"lease_id":"database/creds/analytics_ro/123"}` { @@ -51,6 +62,13 @@ func TestHTTPClient_GenerateCredentialsAndRevokeLease(t *testing.T) { if lease.Username != "dyn-user" || lease.LeaseID != "database/creds/analytics_ro/123" { t.Fatalf("lease = %#v, want parsed vault lease", lease) } + renewed, err := client.RenewLease(context.Background(), lease.LeaseID, 30*time.Minute) + if err != nil { + t.Fatalf("RenewLease() error = %v", err) + } + if renewed.LeaseDuration != 30*time.Minute { + t.Fatalf("renewed lease = %#v, want renewed duration", renewed) + } if err := client.RevokeLease(context.Background(), lease.LeaseID); err != nil { t.Fatalf("RevokeLease() error = %v", err) } diff --git a/internal/connectors/vaultdb/connector.go b/internal/connectors/vaultdb/connector.go index 91b3350..71093c8 100644 --- a/internal/connectors/vaultdb/connector.go +++ b/internal/connectors/vaultdb/connector.go @@ -16,10 +16,12 @@ type LeaseCredentials struct { Password string LeaseID string LeaseDuration time.Duration + Renewable bool } type Client interface { GenerateCredentials(ctx context.Context, role string) (*LeaseCredentials, error) + RenewLease(ctx context.Context, leaseID string, increment time.Duration) (*LeaseCredentials, error) RevokeLease(ctx context.Context, leaseID string) error } @@ -117,6 +119,10 @@ func (c *Connector) Issue(ctx context.Context, req core.IssueRequest) (*core.Iss if err != nil { return nil, err } + lease, leaseExpiresAt, err := c.extendLeaseForGrant(ctx, lease, req.Grant.ExpiresAt) + if err != nil { + return nil, err + } dsn, err := renderDSN(c.roleDSNs[req.Resource.Name], lease) if err != nil { return nil, err @@ -125,16 +131,17 @@ func (c *Connector) Issue(ctx context.Context, req core.IssueRequest) (*core.Iss return &core.IssuedArtifact{ Kind: core.ArtifactKindWrappedSecret, Metadata: map[string]string{ - "artifact_id": "art_" + req.Grant.ID, - "lease_id": lease.LeaseID, - "db_role": req.Resource.Name, + "artifact_id": "art_" + req.Grant.ID, + "lease_id": lease.LeaseID, + "lease_expires_at": leaseExpiresAt.UTC().Format(time.RFC3339), + "db_role": req.Resource.Name, }, SecretData: map[string]string{ "username": lease.Username, "password": lease.Password, "dsn": dsn, }, - ExpiresAt: minTime(req.Grant.ExpiresAt, time.Now().Add(lease.LeaseDuration)), + ExpiresAt: minTime(req.Grant.ExpiresAt, leaseExpiresAt), }, nil } @@ -203,6 +210,50 @@ func renderDSN(renderPattern string, lease *LeaseCredentials) (string, error) { return builder.String(), nil } +func (c *Connector) extendLeaseForGrant(ctx context.Context, lease *LeaseCredentials, grantExpiresAt time.Time) (*LeaseCredentials, time.Time, error) { + now := time.Now() + leaseExpiresAt := now.Add(lease.LeaseDuration) + if grantExpiresAt.IsZero() || !grantExpiresAt.After(leaseExpiresAt) { + return lease, leaseExpiresAt, nil + } + if lease.LeaseID == "" || !lease.Renewable { + return nil, time.Time{}, fmt.Errorf("%w: vault lease expires before the requested grant ttl and is not renewable", core.ErrForbidden) + } + + remaining := time.Until(grantExpiresAt) + renewed, err := c.client.RenewLease(ctx, lease.LeaseID, remaining) + if err != nil { + return nil, time.Time{}, err + } + merged := mergeLeaseCredentials(lease, renewed) + leaseExpiresAt = time.Now().Add(merged.LeaseDuration) + if grantExpiresAt.After(leaseExpiresAt) { + return nil, time.Time{}, fmt.Errorf("%w: vault lease renewal for %q was capped before the requested grant ttl", core.ErrForbidden, lease.LeaseID) + } + return merged, leaseExpiresAt, nil +} + +func mergeLeaseCredentials(current *LeaseCredentials, updated *LeaseCredentials) *LeaseCredentials { + if updated == nil { + return current + } + merged := *current + if updated.Username != "" { + merged.Username = updated.Username + } + if updated.Password != "" { + merged.Password = updated.Password + } + if updated.LeaseID != "" { + merged.LeaseID = updated.LeaseID + } + if updated.LeaseDuration > 0 { + merged.LeaseDuration = updated.LeaseDuration + } + merged.Renewable = updated.Renewable + return &merged +} + func minTime(a time.Time, b time.Time) time.Time { if a.IsZero() { return b diff --git a/internal/connectors/vaultdb/connector_test.go b/internal/connectors/vaultdb/connector_test.go index ddce4b0..a0f1f77 100644 --- a/internal/connectors/vaultdb/connector_test.go +++ b/internal/connectors/vaultdb/connector_test.go @@ -21,6 +21,7 @@ func TestConnector_IssueAndRevokeDynamicCredentials(t *testing.T) { Password: "secret:/?#[]@", LeaseID: "database/creds/analytics_ro/123", LeaseDuration: 10 * time.Minute, + Renewable: true, }, } connector, err := vaultdb.NewConnector(vaultdb.Config{ @@ -54,6 +55,9 @@ func TestConnector_IssueAndRevokeDynamicCredentials(t *testing.T) { if issued.SecretData["dsn"] == "" || issued.Metadata["lease_id"] == "" { t.Fatalf("issued artifact = %#v, want dsn and lease id", issued) } + if issued.Metadata["lease_expires_at"] == "" { + t.Fatalf("issued artifact = %#v, want lease expiry metadata", issued) + } if strings.Contains(issued.SecretData["dsn"], "secret:/?#[]@") { t.Fatalf("dsn = %q, want escaped credentials", issued.SecretData["dsn"]) } @@ -114,6 +118,7 @@ func TestConnectorHonorsConfiguredRoleSuffixes(t *testing.T) { Password: "dyn-pass", LeaseID: "lease-1", LeaseDuration: time.Minute, + Renewable: true, }, }, RoleDSNs: map[string]string{ @@ -197,8 +202,97 @@ func TestConnectorIssueEscapesUserinfoWithPercentEncoding(t *testing.T) { } } +func TestConnector_IssueRenewsLeaseToMatchGrantTTL(t *testing.T) { + t.Parallel() + + client := &fakeVaultClient{ + lease: &vaultdb.LeaseCredentials{ + Username: "dyn-user", + Password: "dyn-pass", + LeaseID: "database/creds/analytics_ro/123", + LeaseDuration: time.Minute, + Renewable: true, + }, + renewedLease: &vaultdb.LeaseCredentials{ + LeaseID: "database/creds/analytics_ro/123", + LeaseDuration: 10 * time.Minute, + Renewable: true, + }, + } + connector, err := vaultdb.NewConnector(vaultdb.Config{ + Client: client, + RoleDSNs: map[string]string{ + "analytics_ro": "postgres://{{username}}:{{password}}@db.internal:5432/analytics?sslmode=require", + }, + }) + if err != nil { + t.Fatalf("NewConnector() error = %v", err) + } + + issued, err := connector.Issue(context.Background(), core.IssueRequest{ + Session: &core.Session{ID: "sess_db", TenantID: "t_acme"}, + Grant: &core.Grant{ + ID: "gr_db", + DeliveryMode: core.DeliveryModeWrappedSecret, + ExpiresAt: time.Now().Add(5 * time.Minute), + }, + Resource: core.ResourceDescriptor{ + Kind: core.ResourceKindDBRole, + Name: "analytics_ro", + }, + }) + if err != nil { + t.Fatalf("Issue() error = %v", err) + } + if client.renewedLeaseID != "database/creds/analytics_ro/123" { + t.Fatalf("renewed lease = %q, want expected lease", client.renewedLeaseID) + } + if got := time.Until(issued.ExpiresAt); got < 4*time.Minute { + t.Fatalf("artifact ttl = %s, want renewed lease to cover grant", got) + } +} + +func TestConnector_IssueFailsWhenGrantTTLExceedsNonRenewableLease(t *testing.T) { + t.Parallel() + + connector, err := vaultdb.NewConnector(vaultdb.Config{ + Client: &fakeVaultClient{ + lease: &vaultdb.LeaseCredentials{ + Username: "dyn-user", + Password: "dyn-pass", + LeaseID: "database/creds/analytics_ro/123", + LeaseDuration: time.Minute, + }, + }, + RoleDSNs: map[string]string{ + "analytics_ro": "postgres://{{username}}:{{password}}@db.internal:5432/analytics?sslmode=require", + }, + }) + if err != nil { + t.Fatalf("NewConnector() error = %v", err) + } + + _, err = connector.Issue(context.Background(), core.IssueRequest{ + Session: &core.Session{ID: "sess_db", TenantID: "t_acme"}, + Grant: &core.Grant{ + ID: "gr_db", + DeliveryMode: core.DeliveryModeWrappedSecret, + ExpiresAt: time.Now().Add(5 * time.Minute), + }, + Resource: core.ResourceDescriptor{ + Kind: core.ResourceKindDBRole, + Name: "analytics_ro", + }, + }) + if err == nil || !strings.Contains(err.Error(), "not renewable") { + t.Fatalf("Issue() error = %v, want renewal failure", err) + } +} + type fakeVaultClient struct { lease *vaultdb.LeaseCredentials + renewedLease *vaultdb.LeaseCredentials + renewedLeaseID string revokedLeaseID string } @@ -206,6 +300,14 @@ func (f *fakeVaultClient) GenerateCredentials(context.Context, string) (*vaultdb return f.lease, nil } +func (f *fakeVaultClient) RenewLease(_ context.Context, leaseID string, _ time.Duration) (*vaultdb.LeaseCredentials, error) { + f.renewedLeaseID = leaseID + if f.renewedLease == nil { + return f.lease, nil + } + return f.renewedLease, nil +} + func (f *fakeVaultClient) RevokeLease(_ context.Context, leaseID string) error { f.revokedLeaseID = leaseID return nil From 9ee660374963d015223855e93279f9b2e1ed5eb4 Mon Sep 17 00:00:00 2001 From: Jonathan Haas Date: Wed, 15 Apr 2026 15:39:34 -0700 Subject: [PATCH 2/2] Revoke Vault DB leases on renewal failure --- internal/connectors/vaultdb/connector.go | 6 +++++- internal/connectors/vaultdb/connector_test.go | 20 +++++++++++-------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/internal/connectors/vaultdb/connector.go b/internal/connectors/vaultdb/connector.go index 71093c8..3e110d7 100644 --- a/internal/connectors/vaultdb/connector.go +++ b/internal/connectors/vaultdb/connector.go @@ -119,10 +119,14 @@ func (c *Connector) Issue(ctx context.Context, req core.IssueRequest) (*core.Iss if err != nil { return nil, err } - lease, leaseExpiresAt, err := c.extendLeaseForGrant(ctx, lease, req.Grant.ExpiresAt) + extendedLease, leaseExpiresAt, err := c.extendLeaseForGrant(ctx, lease, req.Grant.ExpiresAt) if err != nil { + if lease != nil && lease.LeaseID != "" { + _ = c.client.RevokeLease(ctx, lease.LeaseID) + } return nil, err } + lease = extendedLease dsn, err := renderDSN(c.roleDSNs[req.Resource.Name], lease) if err != nil { return nil, err diff --git a/internal/connectors/vaultdb/connector_test.go b/internal/connectors/vaultdb/connector_test.go index a0f1f77..a39f7b0 100644 --- a/internal/connectors/vaultdb/connector_test.go +++ b/internal/connectors/vaultdb/connector_test.go @@ -255,15 +255,16 @@ func TestConnector_IssueRenewsLeaseToMatchGrantTTL(t *testing.T) { func TestConnector_IssueFailsWhenGrantTTLExceedsNonRenewableLease(t *testing.T) { t.Parallel() - connector, err := vaultdb.NewConnector(vaultdb.Config{ - Client: &fakeVaultClient{ - lease: &vaultdb.LeaseCredentials{ - Username: "dyn-user", - Password: "dyn-pass", - LeaseID: "database/creds/analytics_ro/123", - LeaseDuration: time.Minute, - }, + client := &fakeVaultClient{ + lease: &vaultdb.LeaseCredentials{ + Username: "dyn-user", + Password: "dyn-pass", + LeaseID: "database/creds/analytics_ro/123", + LeaseDuration: time.Minute, }, + } + connector, err := vaultdb.NewConnector(vaultdb.Config{ + Client: client, RoleDSNs: map[string]string{ "analytics_ro": "postgres://{{username}}:{{password}}@db.internal:5432/analytics?sslmode=require", }, @@ -287,6 +288,9 @@ func TestConnector_IssueFailsWhenGrantTTLExceedsNonRenewableLease(t *testing.T) if err == nil || !strings.Contains(err.Error(), "not renewable") { t.Fatalf("Issue() error = %v, want renewal failure", err) } + if client.revokedLeaseID != "database/creds/analytics_ro/123" { + t.Fatalf("revoked lease = %q, want failed renewal lease to be revoked", client.revokedLeaseID) + } } type fakeVaultClient struct {